--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-05-20T14:13:33.711159Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-15cdb-68844c9fc9-dsxnm namespace: kserve-ci-e2e-test resourceVersion: "20751" uid: 6304b107-7353-4d51-a2c4-7880f1998511 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-15cdb-68844c9fc9-dsxnm to ip-10-0-136-75.ec2.internal metadata: creationTimestamp: "2026-05-20T14:13:33Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T14:13:33Z" name: ensemble-graph-15cdb-68844c9fc9-dsxnm.18b14b90f2eed9be namespace: kserve-ci-e2e-test resourceVersion: "20756" uid: 0918d742-d423-457d-926a-013814a55b02 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-747dc7bb84-jx2kx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:33Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-15cdb-68844c9fc9-dsxnm namespace: kserve-ci-e2e-test resourceVersion: "20753" uid: 6304b107-7353-4d51-a2c4-7880f1998511 kind: Event lastTimestamp: "2026-05-20T14:13:33Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-15cdb-serving-cert" not found' metadata: creationTimestamp: "2026-05-20T14:13:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:13:33Z" name: ensemble-graph-15cdb-68844c9fc9-dsxnm.18b14b9100dfce77 namespace: kserve-ci-e2e-test resourceVersion: "20770" uid: 0bd92475-1eef-4dc9-82dd-564dec396039 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:34Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-15cdb-68844c9fc9-dsxnm namespace: kserve-ci-e2e-test resourceVersion: "20760" uid: 6304b107-7353-4d51-a2c4-7880f1998511 kind: Event lastTimestamp: "2026-05-20T14:13:34Z" message: Add eth0 [10.134.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T14:13:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T14:13:34Z" name: ensemble-graph-15cdb-68844c9fc9-dsxnm.18b14b9130a088e5 namespace: kserve-ci-e2e-test resourceVersion: "20781" uid: 417fb1f5-49ea-4398-b706-0fdf21b6ae70 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-15cdb} kind: Pod name: ensemble-graph-15cdb-68844c9fc9-dsxnm namespace: kserve-ci-e2e-test resourceVersion: "20753" uid: 6304b107-7353-4d51-a2c4-7880f1998511 kind: Event lastTimestamp: "2026-05-20T14:13:34Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:04926a9d7185ff179745ae32ed77ae962904508dbcb7541e9bf600392af6f503" already present on machine metadata: creationTimestamp: "2026-05-20T14:13:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:13:34Z" name: ensemble-graph-15cdb-68844c9fc9-dsxnm.18b14b913215bb85 namespace: kserve-ci-e2e-test resourceVersion: "20783" uid: 3fe1e690-9ba8-4a0e-affb-717a976ff926 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-15cdb} kind: Pod name: ensemble-graph-15cdb-68844c9fc9-dsxnm namespace: kserve-ci-e2e-test resourceVersion: "20753" uid: 6304b107-7353-4d51-a2c4-7880f1998511 kind: Event lastTimestamp: "2026-05-20T14:13:34Z" message: 'Created container: ensemble-graph-15cdb' metadata: creationTimestamp: "2026-05-20T14:13:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:13:34Z" name: ensemble-graph-15cdb-68844c9fc9-dsxnm.18b14b9135187c8f namespace: kserve-ci-e2e-test resourceVersion: "20784" uid: 2ff91e44-07f7-4863-b6b4-44deb849ca8e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-15cdb} kind: Pod name: ensemble-graph-15cdb-68844c9fc9-dsxnm namespace: kserve-ci-e2e-test resourceVersion: "20753" uid: 6304b107-7353-4d51-a2c4-7880f1998511 kind: Event lastTimestamp: "2026-05-20T14:13:34Z" message: Started container ensemble-graph-15cdb metadata: creationTimestamp: "2026-05-20T14:13:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:13:34Z" name: ensemble-graph-15cdb-68844c9fc9-dsxnm.18b14b91358a401e namespace: kserve-ci-e2e-test resourceVersion: "20785" uid: fe9118bb-7759-4053-84d5-2b8f4d783454 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-15cdb} kind: Pod name: ensemble-graph-15cdb-68844c9fc9-dsxnm namespace: kserve-ci-e2e-test resourceVersion: "20753" uid: 6304b107-7353-4d51-a2c4-7880f1998511 kind: Event lastTimestamp: "2026-05-20T14:13:43Z" message: Stopping container ensemble-graph-15cdb metadata: creationTimestamp: "2026-05-20T14:13:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:13:43Z" name: ensemble-graph-15cdb-68844c9fc9-dsxnm.18b14b934b340344 namespace: kserve-ci-e2e-test resourceVersion: "20879" uid: 19aa3b35-f4f1-40e8-8bb4-730bddea0287 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-20T14:13:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-15cdb} kind: Pod name: ensemble-graph-15cdb-68844c9fc9-dsxnm namespace: kserve-ci-e2e-test resourceVersion: "20753" uid: 6304b107-7353-4d51-a2c4-7880f1998511 kind: Event lastTimestamp: "2026-05-20T14:14:11Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-20T14:13:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:14:11Z" name: ensemble-graph-15cdb-68844c9fc9-dsxnm.18b14b93d84ae802 namespace: kserve-ci-e2e-test resourceVersion: "21276" uid: 78bbaa81-0c9d-4480-8c2c-64275ec64709 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:33Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-15cdb-68844c9fc9 namespace: kserve-ci-e2e-test resourceVersion: "20748" uid: d0b8846e-cfd5-4b1f-bd52-8ee073b6a12d kind: Event lastTimestamp: "2026-05-20T14:13:33Z" message: 'Created pod: ensemble-graph-15cdb-68844c9fc9-dsxnm' metadata: creationTimestamp: "2026-05-20T14:13:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:13:33Z" name: ensemble-graph-15cdb-68844c9fc9.18b14b90f27ba9c1 namespace: kserve-ci-e2e-test resourceVersion: "20752" uid: 6fef7060-2bd9-412d-91f6-8979fa93a65b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:33Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-15cdb namespace: kserve-ci-e2e-test resourceVersion: "20746" uid: 8ece6e34-5f14-4d91-b4b0-80d6ebfd0db5 kind: Event lastTimestamp: "2026-05-20T14:13:33Z" message: Scaled up replica set ensemble-graph-15cdb-68844c9fc9 from 0 to 1 metadata: creationTimestamp: "2026-05-20T14:13:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:13:33Z" name: ensemble-graph-15cdb.18b14b90f1a487d6 namespace: kserve-ci-e2e-test resourceVersion: "20749" uid: d058c272-0634-44dd-bc4e-5f4471750bb3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:33Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-15cdb namespace: kserve-ci-e2e-test resourceVersion: "20764" uid: 21b63d16-18c5-44fb-94e9-01cb8f56b6d0 kind: Event lastTimestamp: "2026-05-20T14:13:33Z" message: InferenceGraph [ensemble-graph-15cdb] is Ready metadata: creationTimestamp: "2026-05-20T14:13:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:13:33Z" name: ensemble-graph-15cdb.18b14b90f7dff2c7 namespace: kserve-ci-e2e-test resourceVersion: "20766" uid: c0183a2d-cf10-4363-9709-627606bfd241 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T14:14:44.004777Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-ecafe-fd8585c74-tmpzq namespace: kserve-ci-e2e-test resourceVersion: "21727" uid: ffa3794a-a4aa-43fe-a2f5-e6d3a576ee5d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-ecafe-fd8585c74-tmpzq to ip-10-0-136-75.ec2.internal metadata: creationTimestamp: "2026-05-20T14:14:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T14:14:44Z" name: ensemble-graph-ecafe-fd8585c74-tmpzq.18b14ba150c24df4 namespace: kserve-ci-e2e-test resourceVersion: "21733" uid: 766db8f3-5489-46d4-b214-dac099c2f9a4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-747dc7bb84-jx2kx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:14:44Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-ecafe-fd8585c74-tmpzq namespace: kserve-ci-e2e-test resourceVersion: "21729" uid: ffa3794a-a4aa-43fe-a2f5-e6d3a576ee5d kind: Event lastTimestamp: "2026-05-20T14:14:44Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-ecafe-serving-cert" not found' metadata: creationTimestamp: "2026-05-20T14:14:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:14:44Z" name: ensemble-graph-ecafe-fd8585c74-tmpzq.18b14ba1597f5840 namespace: kserve-ci-e2e-test resourceVersion: "21746" uid: a3eec902-710c-4527-9b8b-7e545e41ef7f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:14:45Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-ecafe-fd8585c74-tmpzq namespace: kserve-ci-e2e-test resourceVersion: "21736" uid: ffa3794a-a4aa-43fe-a2f5-e6d3a576ee5d kind: Event lastTimestamp: "2026-05-20T14:14:45Z" message: Add eth0 [10.134.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T14:14:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T14:14:45Z" name: ensemble-graph-ecafe-fd8585c74-tmpzq.18b14ba18e3da43f namespace: kserve-ci-e2e-test resourceVersion: "21758" uid: af2f41ce-1def-49a7-94ea-7dfbc75b8d28 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:14:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-ecafe} kind: Pod name: ensemble-graph-ecafe-fd8585c74-tmpzq namespace: kserve-ci-e2e-test resourceVersion: "21729" uid: ffa3794a-a4aa-43fe-a2f5-e6d3a576ee5d kind: Event lastTimestamp: "2026-05-20T14:14:45Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:04926a9d7185ff179745ae32ed77ae962904508dbcb7541e9bf600392af6f503" already present on machine metadata: creationTimestamp: "2026-05-20T14:14:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:14:45Z" name: ensemble-graph-ecafe-fd8585c74-tmpzq.18b14ba190cc2003 namespace: kserve-ci-e2e-test resourceVersion: "21762" uid: ed3407aa-9fcd-4882-8958-c90262bc3626 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:14:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-ecafe} kind: Pod name: ensemble-graph-ecafe-fd8585c74-tmpzq namespace: kserve-ci-e2e-test resourceVersion: "21729" uid: ffa3794a-a4aa-43fe-a2f5-e6d3a576ee5d kind: Event lastTimestamp: "2026-05-20T14:14:45Z" message: 'Created container: ensemble-graph-ecafe' metadata: creationTimestamp: "2026-05-20T14:14:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:14:45Z" name: ensemble-graph-ecafe-fd8585c74-tmpzq.18b14ba19420b07f namespace: kserve-ci-e2e-test resourceVersion: "21764" uid: 2156f6f0-b54a-415f-acd3-6eb6aafcf34d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:14:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-ecafe} kind: Pod name: ensemble-graph-ecafe-fd8585c74-tmpzq namespace: kserve-ci-e2e-test resourceVersion: "21729" uid: ffa3794a-a4aa-43fe-a2f5-e6d3a576ee5d kind: Event lastTimestamp: "2026-05-20T14:14:45Z" message: Started container ensemble-graph-ecafe metadata: creationTimestamp: "2026-05-20T14:14:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:14:45Z" name: ensemble-graph-ecafe-fd8585c74-tmpzq.18b14ba1949b9c6c namespace: kserve-ci-e2e-test resourceVersion: "21765" uid: 0f31d553-924a-4353-9a38-69197c9fc5e1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:22:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-ecafe} kind: Pod name: ensemble-graph-ecafe-fd8585c74-tmpzq namespace: kserve-ci-e2e-test resourceVersion: "21729" uid: ffa3794a-a4aa-43fe-a2f5-e6d3a576ee5d kind: Event lastTimestamp: "2026-05-20T14:22:58Z" message: Stopping container ensemble-graph-ecafe metadata: creationTimestamp: "2026-05-20T14:22:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:22:58Z" name: ensemble-graph-ecafe-fd8585c74-tmpzq.18b14c1478dcb69e namespace: kserve-ci-e2e-test resourceVersion: "25240" uid: 53b026eb-c2fa-4ed0-9e2f-4a399828c913 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-20T14:23:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-ecafe} kind: Pod name: ensemble-graph-ecafe-fd8585c74-tmpzq namespace: kserve-ci-e2e-test resourceVersion: "21729" uid: ffa3794a-a4aa-43fe-a2f5-e6d3a576ee5d kind: Event lastTimestamp: "2026-05-20T14:23:26Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-20T14:23:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:23:26Z" name: ensemble-graph-ecafe-fd8585c74-tmpzq.18b14c1520c2343e namespace: kserve-ci-e2e-test resourceVersion: "25576" uid: 2fcf38b3-4640-4e55-9988-fa6aa5030db3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:14:43Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-ecafe-fd8585c74 namespace: kserve-ci-e2e-test resourceVersion: "21724" uid: bff0b771-24dc-4c48-b905-3fff4ac6cf04 kind: Event lastTimestamp: "2026-05-20T14:14:43Z" message: 'Created pod: ensemble-graph-ecafe-fd8585c74-tmpzq' metadata: creationTimestamp: "2026-05-20T14:14:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:14:43Z" name: ensemble-graph-ecafe-fd8585c74.18b14ba1503d302d namespace: kserve-ci-e2e-test resourceVersion: "21728" uid: f17165e8-25c2-40d2-92a2-47384c4a8f85 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:14:43Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-ecafe namespace: kserve-ci-e2e-test resourceVersion: "21720" uid: 376c544b-eba9-4059-93a4-100d827df1ad kind: Event lastTimestamp: "2026-05-20T14:14:43Z" message: Scaled up replica set ensemble-graph-ecafe-fd8585c74 from 0 to 1 metadata: creationTimestamp: "2026-05-20T14:14:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:14:43Z" name: ensemble-graph-ecafe.18b14ba14f2b77f3 namespace: kserve-ci-e2e-test resourceVersion: "21725" uid: 600973b3-2e86-4892-927f-b7c34d567de9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:14:44Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-ecafe namespace: kserve-ci-e2e-test resourceVersion: "21738" uid: bed0d0a9-ebfb-422d-9331-155e8730fbae kind: Event lastTimestamp: "2026-05-20T14:14:44Z" message: InferenceGraph [ensemble-graph-ecafe] is Ready metadata: creationTimestamp: "2026-05-20T14:14:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:14:44Z" name: ensemble-graph-ecafe.18b14ba155ac1027 namespace: kserve-ci-e2e-test resourceVersion: "21740" uid: 965998f2-0677-409b-97c2-fdcb14741694 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T14:14:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-ecafe namespace: kserve-ci-e2e-test resourceVersion: "21722" uid: a7f153f4-18df-457b-9bd3-b691a12c1595 kind: Event lastTimestamp: "2026-05-20T14:15:29Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:14:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:15:29Z" name: ensemble-graph-ecafe.18b14ba4cdb68f3b namespace: kserve-ci-e2e-test resourceVersion: "22166" uid: 495116f6-f1e3-4fba-ab2b-1b21261ac8d9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T14:14:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-ecafe namespace: kserve-ci-e2e-test resourceVersion: "21722" uid: a7f153f4-18df-457b-9bd3-b691a12c1595 kind: Event lastTimestamp: "2026-05-20T14:15:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:14:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:15:29Z" name: ensemble-graph-ecafe.18b14ba4ce130250 namespace: kserve-ci-e2e-test resourceVersion: "22167" uid: f409219b-bd3b-4f0b-bac7-bb48919b08a4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-05-20T14:13:03.691337Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-14a90-predictor-547774646f-lrxtg namespace: kserve-ci-e2e-test resourceVersion: "20433" uid: 677266f2-8375-4dcf-a101-0160c6f8f5b4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-14a90-predictor-547774646f-lrxtg to ip-10-0-136-75.ec2.internal metadata: creationTimestamp: "2026-05-20T14:13:03Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T14:13:03Z" name: error-404-isvc-14a90-predictor-547774646f-lrxtg.18b14b89f59cb965 namespace: kserve-ci-e2e-test resourceVersion: "20437" uid: d8b81385-8daf-40ee-931b-64fa934a9b75 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-747dc7bb84-jx2kx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:03Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-14a90-predictor-547774646f-lrxtg namespace: kserve-ci-e2e-test resourceVersion: "20434" uid: 677266f2-8375-4dcf-a101-0160c6f8f5b4 kind: Event lastTimestamp: "2026-05-20T14:13:03Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-14a90-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-20T14:13:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:13:03Z" name: error-404-isvc-14a90-predictor-547774646f-lrxtg.18b14b8a002f04f5 namespace: kserve-ci-e2e-test resourceVersion: "20450" uid: f03e4906-a49a-4b7f-95d1-8042ab5f1f35 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:04Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-14a90-predictor-547774646f-lrxtg namespace: kserve-ci-e2e-test resourceVersion: "20446" uid: 677266f2-8375-4dcf-a101-0160c6f8f5b4 kind: Event lastTimestamp: "2026-05-20T14:13:04Z" message: Add eth0 [10.134.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T14:13:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T14:13:04Z" name: error-404-isvc-14a90-predictor-547774646f-lrxtg.18b14b8a32e973d2 namespace: kserve-ci-e2e-test resourceVersion: "20472" uid: 400649c3-6e06-4950-b9af-c3653b8bb204 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-14a90-predictor-547774646f-lrxtg namespace: kserve-ci-e2e-test resourceVersion: "20434" uid: 677266f2-8375-4dcf-a101-0160c6f8f5b4 kind: Event lastTimestamp: "2026-05-20T14:13:04Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1501" already present on machine metadata: creationTimestamp: "2026-05-20T14:13:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:13:04Z" name: error-404-isvc-14a90-predictor-547774646f-lrxtg.18b14b8a348fc58f namespace: kserve-ci-e2e-test resourceVersion: "20475" uid: 12fc714a-ac39-4487-a5d0-1d7e1a7cf48f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-14a90-predictor-547774646f-lrxtg namespace: kserve-ci-e2e-test resourceVersion: "20434" uid: 677266f2-8375-4dcf-a101-0160c6f8f5b4 kind: Event lastTimestamp: "2026-05-20T14:13:04Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T14:13:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:13:04Z" name: error-404-isvc-14a90-predictor-547774646f-lrxtg.18b14b8a387c0f02 namespace: kserve-ci-e2e-test resourceVersion: "20476" uid: 72509d52-5272-4024-afdc-6b4ba5405952 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-14a90-predictor-547774646f-lrxtg namespace: kserve-ci-e2e-test resourceVersion: "20434" uid: 677266f2-8375-4dcf-a101-0160c6f8f5b4 kind: Event lastTimestamp: "2026-05-20T14:13:04Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T14:13:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:13:04Z" name: error-404-isvc-14a90-predictor-547774646f-lrxtg.18b14b8a38e11675 namespace: kserve-ci-e2e-test resourceVersion: "20477" uid: 0760f7e5-1ae4-4bde-8705-fe296fba1f8f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-14a90-predictor-547774646f-lrxtg namespace: kserve-ci-e2e-test resourceVersion: "20434" uid: 677266f2-8375-4dcf-a101-0160c6f8f5b4 kind: Event lastTimestamp: "2026-05-20T14:13:04Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-20T14:13:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:13:04Z" name: error-404-isvc-14a90-predictor-547774646f-lrxtg.18b14b8a38ee75d1 namespace: kserve-ci-e2e-test resourceVersion: "20478" uid: 0a4bea31-852d-467c-8239-3bbf0950d09a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-14a90-predictor-547774646f-lrxtg namespace: kserve-ci-e2e-test resourceVersion: "20434" uid: 677266f2-8375-4dcf-a101-0160c6f8f5b4 kind: Event lastTimestamp: "2026-05-20T14:13:04Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T14:13:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:13:04Z" name: error-404-isvc-14a90-predictor-547774646f-lrxtg.18b14b8a3c524008 namespace: kserve-ci-e2e-test resourceVersion: "20479" uid: 5520cead-b018-498e-a9bd-c4e641a4bebb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-14a90-predictor-547774646f-lrxtg namespace: kserve-ci-e2e-test resourceVersion: "20434" uid: 677266f2-8375-4dcf-a101-0160c6f8f5b4 kind: Event lastTimestamp: "2026-05-20T14:13:04Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:13:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:13:04Z" name: error-404-isvc-14a90-predictor-547774646f-lrxtg.18b14b8a3cc28f3b namespace: kserve-ci-e2e-test resourceVersion: "20480" uid: 4b04c1e5-34b0-4a62-99ed-cdc1b5fbde43 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-20T14:13:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-14a90-predictor-547774646f-lrxtg namespace: kserve-ci-e2e-test resourceVersion: "20434" uid: 677266f2-8375-4dcf-a101-0160c6f8f5b4 kind: Event lastTimestamp: "2026-05-20T14:13:42Z" message: 'Readiness probe failed: dial tcp 10.134.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T14:13:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:13:42Z" name: error-404-isvc-14a90-predictor-547774646f-lrxtg.18b14b8a815a1948 namespace: kserve-ci-e2e-test resourceVersion: "20846" uid: aeefbd15-b1d2-4fb0-8099-d3bccd547a7b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:14:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-14a90-predictor-547774646f-lrxtg namespace: kserve-ci-e2e-test resourceVersion: "20434" uid: 677266f2-8375-4dcf-a101-0160c6f8f5b4 kind: Event lastTimestamp: "2026-05-20T14:14:13Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T14:14:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:14:13Z" name: error-404-isvc-14a90-predictor-547774646f-lrxtg.18b14b9a3e829054 namespace: kserve-ci-e2e-test resourceVersion: "21344" uid: c306b223-ccee-4667-875e-8397ee8e4072 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:14:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-14a90-predictor-547774646f-lrxtg namespace: kserve-ci-e2e-test resourceVersion: "20434" uid: 677266f2-8375-4dcf-a101-0160c6f8f5b4 kind: Event lastTimestamp: "2026-05-20T14:14:13Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:14:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:14:13Z" name: error-404-isvc-14a90-predictor-547774646f-lrxtg.18b14b9a3e83c880 namespace: kserve-ci-e2e-test resourceVersion: "21347" uid: 0be3ab05-5147-4682-88db-f451ac8ba4f4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:14:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-14a90-predictor-547774646f-lrxtg namespace: kserve-ci-e2e-test resourceVersion: "20434" uid: 677266f2-8375-4dcf-a101-0160c6f8f5b4 kind: Event lastTimestamp: "2026-05-20T14:14:18Z" message: 'Readiness probe failed: Get "https://10.134.0.39:8643/healthz": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)' metadata: creationTimestamp: "2026-05-20T14:14:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:14:18Z" name: error-404-isvc-14a90-predictor-547774646f-lrxtg.18b14b9b452800ae namespace: kserve-ci-e2e-test resourceVersion: "21487" uid: 56b1b149-f9f0-4475-ada3-7bc7053779eb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:03Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-14a90-predictor-547774646f namespace: kserve-ci-e2e-test resourceVersion: "20425" uid: 9bb9cee8-5744-4cb2-96b0-42aae7a7ee1c kind: Event lastTimestamp: "2026-05-20T14:13:03Z" message: 'Created pod: error-404-isvc-14a90-predictor-547774646f-lrxtg' metadata: creationTimestamp: "2026-05-20T14:13:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:13:03Z" name: error-404-isvc-14a90-predictor-547774646f.18b14b89f517da49 namespace: kserve-ci-e2e-test resourceVersion: "20435" uid: c19b4f8a-788c-4e57-b5a3-ce15b03bc472 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:03Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-14a90-predictor namespace: kserve-ci-e2e-test resourceVersion: "20420" uid: 9b620d67-80b9-410e-ae7a-f2766c16e73a kind: Event lastTimestamp: "2026-05-20T14:13:03Z" message: Scaled up replica set error-404-isvc-14a90-predictor-547774646f from 0 to 1 metadata: creationTimestamp: "2026-05-20T14:13:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:13:03Z" name: error-404-isvc-14a90-predictor.18b14b89f22134f5 namespace: kserve-ci-e2e-test resourceVersion: "20428" uid: fec3665d-fd2c-42ed-8d41-616cbcf9bfea reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T14:13:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-14a90-predictor namespace: kserve-ci-e2e-test resourceVersion: "20423" uid: bf77adf9-b130-4282-9b7d-00972578f7dc kind: Event lastTimestamp: "2026-05-20T14:13:33Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:13:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:13:33Z" name: error-404-isvc-14a90-predictor.18b14b8d70a2cec6 namespace: kserve-ci-e2e-test resourceVersion: "20740" uid: c0fc5b5e-cf1d-4342-8b00-68d5cd2ee2e9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T14:13:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-14a90-predictor namespace: kserve-ci-e2e-test resourceVersion: "20423" uid: bf77adf9-b130-4282-9b7d-00972578f7dc kind: Event lastTimestamp: "2026-05-20T14:13:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:13:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:13:33Z" name: error-404-isvc-14a90-predictor.18b14b8d7102fb01 namespace: kserve-ci-e2e-test resourceVersion: "20743" uid: 6c4eba17-85fb-4646-9b3d-f3e7c030d260 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T14:13:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-14a90-predictor namespace: kserve-ci-e2e-test resourceVersion: "20607" uid: bf77adf9-b130-4282-9b7d-00972578f7dc kind: Event lastTimestamp: "2026-05-20T14:14:03Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:13:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:14:03Z" name: error-404-isvc-14a90-predictor.18b14b946ea90110 namespace: kserve-ci-e2e-test resourceVersion: "21206" uid: eb2fa3d0-9fd5-421d-9c09-70483cfcfb4b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T14:13:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-14a90-predictor namespace: kserve-ci-e2e-test resourceVersion: "20607" uid: bf77adf9-b130-4282-9b7d-00972578f7dc kind: Event lastTimestamp: "2026-05-20T14:14:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:13:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:14:03Z" name: error-404-isvc-14a90-predictor.18b14b946f6478db namespace: kserve-ci-e2e-test resourceVersion: "21207" uid: 7e9bb39c-f9e4-422d-b53c-c9943b78e873 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-20T14:13:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-14a90 namespace: kserve-ci-e2e-test resourceVersion: "20389" uid: 12e060e6-d33b-4302-8d99-33f5df123751 kind: Event lastTimestamp: "2026-05-20T14:13:52Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T14:13:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:13:52Z" name: error-404-isvc-14a90.18b14b89f1bb4198 namespace: kserve-ci-e2e-test resourceVersion: "21091" uid: ef6e7e19-bbd8-40cc-8980-f6e7f7d166c3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-14a90 namespace: kserve-ci-e2e-test resourceVersion: "20389" uid: 12e060e6-d33b-4302-8d99-33f5df123751 kind: Event lastTimestamp: "2026-05-20T14:13:03Z" message: 'Failed to update status for InferenceService "error-404-isvc-14a90": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-14a90": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T14:13:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:13:03Z" name: error-404-isvc-14a90.18b14b89f299c44c namespace: kserve-ci-e2e-test resourceVersion: "20430" uid: f499a89a-8706-4eba-983a-95a157ed7874 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-14a90 namespace: kserve-ci-e2e-test resourceVersion: "20389" uid: 12e060e6-d33b-4302-8d99-33f5df123751 kind: Event lastTimestamp: "2026-05-20T14:13:03Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-14a90": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T14:13:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:13:03Z" name: error-404-isvc-14a90.18b14b89f29a153e namespace: kserve-ci-e2e-test resourceVersion: "20431" uid: 80d8eb9d-e8d4-4227-a9d7-3229ebf41707 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-14a90 namespace: kserve-ci-e2e-test resourceVersion: "21087" uid: 12e060e6-d33b-4302-8d99-33f5df123751 kind: Event lastTimestamp: "2026-05-20T14:13:52Z" message: InferenceService [error-404-isvc-14a90] is Ready metadata: creationTimestamp: "2026-05-20T14:13:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:13:52Z" name: error-404-isvc-14a90.18b14b953cd4e2c1 namespace: kserve-ci-e2e-test resourceVersion: "21088" uid: ab092d21-36e3-42f6-aecd-8eff5e3ce434 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T14:12:33.973601Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-15cdb-predictor-7c646df98f-sjwvp namespace: kserve-ci-e2e-test resourceVersion: "20072" uid: e93a15ed-cdfb-4bb2-b08f-935716638b5a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-15cdb-predictor-7c646df98f-sjwvp to ip-10-0-136-75.ec2.internal metadata: creationTimestamp: "2026-05-20T14:12:33Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T14:12:33Z" name: error-404-isvc-15cdb-predictor-7c646df98f-sjwvp.18b14b830a4c02c7 namespace: kserve-ci-e2e-test resourceVersion: "20076" uid: 71890771-31d8-41b3-a170-993467764192 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-747dc7bb84-jx2kx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:12:34Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-15cdb-predictor-7c646df98f-sjwvp namespace: kserve-ci-e2e-test resourceVersion: "20085" uid: e93a15ed-cdfb-4bb2-b08f-935716638b5a kind: Event lastTimestamp: "2026-05-20T14:12:34Z" message: Add eth0 [10.134.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T14:12:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T14:12:34Z" name: error-404-isvc-15cdb-predictor-7c646df98f-sjwvp.18b14b83242102dd namespace: kserve-ci-e2e-test resourceVersion: "20098" uid: 7c88b756-3d1e-413f-b01c-174f5a3f8eb2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:12:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-15cdb-predictor-7c646df98f-sjwvp namespace: kserve-ci-e2e-test resourceVersion: "20075" uid: e93a15ed-cdfb-4bb2-b08f-935716638b5a kind: Event lastTimestamp: "2026-05-20T14:12:34Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1501" already present on machine metadata: creationTimestamp: "2026-05-20T14:12:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:12:34Z" name: error-404-isvc-15cdb-predictor-7c646df98f-sjwvp.18b14b83268c85c6 namespace: kserve-ci-e2e-test resourceVersion: "20101" uid: a49b6c41-6435-4778-b9ce-32170f81ad3e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:12:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-15cdb-predictor-7c646df98f-sjwvp namespace: kserve-ci-e2e-test resourceVersion: "20075" uid: e93a15ed-cdfb-4bb2-b08f-935716638b5a kind: Event lastTimestamp: "2026-05-20T14:12:34Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T14:12:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:12:34Z" name: error-404-isvc-15cdb-predictor-7c646df98f-sjwvp.18b14b8329a07762 namespace: kserve-ci-e2e-test resourceVersion: "20102" uid: fc77c146-12da-4535-9e08-38448094c630 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:12:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-15cdb-predictor-7c646df98f-sjwvp namespace: kserve-ci-e2e-test resourceVersion: "20075" uid: e93a15ed-cdfb-4bb2-b08f-935716638b5a kind: Event lastTimestamp: "2026-05-20T14:12:34Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T14:12:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:12:34Z" name: error-404-isvc-15cdb-predictor-7c646df98f-sjwvp.18b14b832a0e32a6 namespace: kserve-ci-e2e-test resourceVersion: "20103" uid: 50e3069c-70d3-4dd2-90e1-80ff9b928aad reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:12:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-15cdb-predictor-7c646df98f-sjwvp namespace: kserve-ci-e2e-test resourceVersion: "20075" uid: e93a15ed-cdfb-4bb2-b08f-935716638b5a kind: Event lastTimestamp: "2026-05-20T14:12:34Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-20T14:12:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:12:34Z" name: error-404-isvc-15cdb-predictor-7c646df98f-sjwvp.18b14b832a1afe96 namespace: kserve-ci-e2e-test resourceVersion: "20104" uid: e1360896-5bc1-4c26-aa9b-b59747f5de34 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:12:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-15cdb-predictor-7c646df98f-sjwvp namespace: kserve-ci-e2e-test resourceVersion: "20075" uid: e93a15ed-cdfb-4bb2-b08f-935716638b5a kind: Event lastTimestamp: "2026-05-20T14:12:34Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T14:12:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:12:34Z" name: error-404-isvc-15cdb-predictor-7c646df98f-sjwvp.18b14b832e87a8c9 namespace: kserve-ci-e2e-test resourceVersion: "20107" uid: bb915751-192d-4c34-9a15-150395c0b107 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:12:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-15cdb-predictor-7c646df98f-sjwvp namespace: kserve-ci-e2e-test resourceVersion: "20075" uid: e93a15ed-cdfb-4bb2-b08f-935716638b5a kind: Event lastTimestamp: "2026-05-20T14:12:34Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:12:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:12:34Z" name: error-404-isvc-15cdb-predictor-7c646df98f-sjwvp.18b14b832ef54a54 namespace: kserve-ci-e2e-test resourceVersion: "20109" uid: abaae99c-aa97-4def-9396-b890e5502aa2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-20T14:12:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-15cdb-predictor-7c646df98f-sjwvp namespace: kserve-ci-e2e-test resourceVersion: "20075" uid: e93a15ed-cdfb-4bb2-b08f-935716638b5a kind: Event lastTimestamp: "2026-05-20T14:13:10Z" message: 'Readiness probe failed: dial tcp 10.134.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T14:12:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:13:10Z" name: error-404-isvc-15cdb-predictor-7c646df98f-sjwvp.18b14b8342730c68 namespace: kserve-ci-e2e-test resourceVersion: "20535" uid: e0886803-f81d-4b5d-9161-624d154b521e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-15cdb-predictor-7c646df98f-sjwvp namespace: kserve-ci-e2e-test resourceVersion: "20075" uid: e93a15ed-cdfb-4bb2-b08f-935716638b5a kind: Event lastTimestamp: "2026-05-20T14:13:44Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T14:13:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:13:44Z" name: error-404-isvc-15cdb-predictor-7c646df98f-sjwvp.18b14b93587449c4 namespace: kserve-ci-e2e-test resourceVersion: "20946" uid: c625aba1-8d5e-457d-acc0-dc0845082111 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-15cdb-predictor-7c646df98f-sjwvp namespace: kserve-ci-e2e-test resourceVersion: "20075" uid: e93a15ed-cdfb-4bb2-b08f-935716638b5a kind: Event lastTimestamp: "2026-05-20T14:13:44Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:13:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:13:44Z" name: error-404-isvc-15cdb-predictor-7c646df98f-sjwvp.18b14b935876c6d2 namespace: kserve-ci-e2e-test resourceVersion: "20950" uid: 0c9a344f-f0e5-4047-bb76-77a26089d64f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-15cdb-predictor-7c646df98f-sjwvp namespace: kserve-ci-e2e-test resourceVersion: "20075" uid: e93a15ed-cdfb-4bb2-b08f-935716638b5a kind: Event lastTimestamp: "2026-05-20T14:13:45Z" message: 'Readiness probe failed: Get "https://10.134.0.38:8643/healthz": dial tcp 10.134.0.38:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-20T14:13:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:13:45Z" name: error-404-isvc-15cdb-predictor-7c646df98f-sjwvp.18b14b93cab2bb64 namespace: kserve-ci-e2e-test resourceVersion: "21012" uid: 3d90e5ec-b99e-4a48-b591-c3b2819cb3d4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:12:33Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-15cdb-predictor-7c646df98f namespace: kserve-ci-e2e-test resourceVersion: "20060" uid: f504c7a0-78fc-47b9-8ca5-802503491734 kind: Event lastTimestamp: "2026-05-20T14:12:33Z" message: 'Created pod: error-404-isvc-15cdb-predictor-7c646df98f-sjwvp' metadata: creationTimestamp: "2026-05-20T14:12:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:12:33Z" name: error-404-isvc-15cdb-predictor-7c646df98f.18b14b830850bc71 namespace: kserve-ci-e2e-test resourceVersion: "20073" uid: c0b8e310-4134-40ed-95f3-1465767c7e4d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:12:33Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-15cdb-predictor namespace: kserve-ci-e2e-test resourceVersion: "20052" uid: 2d5dc132-887d-4640-8e44-aa57c1120d04 kind: Event lastTimestamp: "2026-05-20T14:12:33Z" message: Scaled up replica set error-404-isvc-15cdb-predictor-7c646df98f from 0 to 1 metadata: creationTimestamp: "2026-05-20T14:12:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:12:33Z" name: error-404-isvc-15cdb-predictor.18b14b830431562f namespace: kserve-ci-e2e-test resourceVersion: "20065" uid: f012557b-9e5c-493e-8b68-12beb9c6363e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T14:12:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-15cdb-predictor namespace: kserve-ci-e2e-test resourceVersion: "20059" uid: 0ce963e2-3772-4ccf-aa21-7afa6ff5d715 kind: Event lastTimestamp: "2026-05-20T14:13:33Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:12:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:13:33Z" name: error-404-isvc-15cdb-predictor.18b14b868318a44d namespace: kserve-ci-e2e-test resourceVersion: "20771" uid: 6c01332d-e7dc-4de1-a0ad-039425793de7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T14:12:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-15cdb-predictor namespace: kserve-ci-e2e-test resourceVersion: "20059" uid: 0ce963e2-3772-4ccf-aa21-7afa6ff5d715 kind: Event lastTimestamp: "2026-05-20T14:13:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:12:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:13:33Z" name: error-404-isvc-15cdb-predictor.18b14b8683b5c74a namespace: kserve-ci-e2e-test resourceVersion: "20772" uid: 808dc8c6-4541-4714-9011-0198ac4dc0d3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-15cdb-predictor namespace: kserve-ci-e2e-test resourceVersion: "20228" uid: 0ce963e2-3772-4ccf-aa21-7afa6ff5d715 kind: Event lastTimestamp: "2026-05-20T14:13:03Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:13:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:13:03Z" name: error-404-isvc-15cdb-predictor.18b14b8a02e4e116 namespace: kserve-ci-e2e-test resourceVersion: "20457" uid: e96be21c-d74a-4401-b938-01dda8e683b1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-15cdb-predictor namespace: kserve-ci-e2e-test resourceVersion: "20228" uid: 0ce963e2-3772-4ccf-aa21-7afa6ff5d715 kind: Event lastTimestamp: "2026-05-20T14:13:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:13:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:13:03Z" name: error-404-isvc-15cdb-predictor.18b14b8a03615196 namespace: kserve-ci-e2e-test resourceVersion: "20458" uid: 7c1590d2-c6f5-47c9-aa5d-83ec76ba59ff reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-20T14:12:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-15cdb namespace: kserve-ci-e2e-test resourceVersion: "20028" uid: 1c70bca8-13b2-410d-b0c2-61fc32eac469 kind: Event lastTimestamp: "2026-05-20T14:13:21Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T14:12:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:13:21Z" name: error-404-isvc-15cdb.18b14b8303934d2d namespace: kserve-ci-e2e-test resourceVersion: "20648" uid: 2effd739-478a-47d8-a103-ab27a1ec3536 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:12:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-15cdb namespace: kserve-ci-e2e-test resourceVersion: "20028" uid: 1c70bca8-13b2-410d-b0c2-61fc32eac469 kind: Event lastTimestamp: "2026-05-20T14:12:33Z" message: 'Failed to update status for InferenceService "error-404-isvc-15cdb": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-15cdb": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T14:12:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:12:33Z" name: error-404-isvc-15cdb.18b14b83047e36a7 namespace: kserve-ci-e2e-test resourceVersion: "20066" uid: e5eb77a8-1a8e-41e3-92b9-60041f47d544 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:12:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-15cdb namespace: kserve-ci-e2e-test resourceVersion: "20028" uid: 1c70bca8-13b2-410d-b0c2-61fc32eac469 kind: Event lastTimestamp: "2026-05-20T14:12:33Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-15cdb": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T14:12:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:12:33Z" name: error-404-isvc-15cdb.18b14b83047e9925 namespace: kserve-ci-e2e-test resourceVersion: "20068" uid: 4fb3b903-e61c-4a91-879b-6580829fb2a3 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-15cdb namespace: kserve-ci-e2e-test resourceVersion: "20644" uid: 1c70bca8-13b2-410d-b0c2-61fc32eac469 kind: Event lastTimestamp: "2026-05-20T14:13:21Z" message: InferenceService [error-404-isvc-15cdb] is Ready metadata: creationTimestamp: "2026-05-20T14:13:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:13:21Z" name: error-404-isvc-15cdb.18b14b8e01747e4e namespace: kserve-ci-e2e-test resourceVersion: "20646" uid: ddbe916d-7b0f-4ee2-9cb6-831a1eb29175 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T14:24:09.259688Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-31b2e-predictor-848cf47897-zpj2x namespace: kserve-ci-e2e-test resourceVersion: "26513" uid: 247807be-896a-45f3-9ae9-5266e0b9fa63 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-31b2e-predictor-848cf47897-zpj2x to ip-10-0-136-75.ec2.internal metadata: creationTimestamp: "2026-05-20T14:24:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T14:24:09Z" name: error-404-isvc-31b2e-predictor-848cf47897-zpj2x.18b14c24ec93c6fb namespace: kserve-ci-e2e-test resourceVersion: "26518" uid: 4b528606-56d0-4c70-95db-693161a8be5d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-747dc7bb84-jx2kx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:24:09Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-31b2e-predictor-848cf47897-zpj2x namespace: kserve-ci-e2e-test resourceVersion: "26524" uid: 247807be-896a-45f3-9ae9-5266e0b9fa63 kind: Event lastTimestamp: "2026-05-20T14:24:09Z" message: Add eth0 [10.134.0.49/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T14:24:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T14:24:09Z" name: error-404-isvc-31b2e-predictor-848cf47897-zpj2x.18b14c2506adb23e namespace: kserve-ci-e2e-test resourceVersion: "26534" uid: faeba3a3-2fcc-47c2-8df7-0a99473df9a3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:24:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-31b2e-predictor-848cf47897-zpj2x namespace: kserve-ci-e2e-test resourceVersion: "26516" uid: 247807be-896a-45f3-9ae9-5266e0b9fa63 kind: Event lastTimestamp: "2026-05-20T14:24:09Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1501" already present on machine metadata: creationTimestamp: "2026-05-20T14:24:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:24:09Z" name: error-404-isvc-31b2e-predictor-848cf47897-zpj2x.18b14c250870d82d namespace: kserve-ci-e2e-test resourceVersion: "26536" uid: 32fbc7d0-5160-41b3-b77a-f738e40403fc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:24:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-31b2e-predictor-848cf47897-zpj2x namespace: kserve-ci-e2e-test resourceVersion: "26516" uid: 247807be-896a-45f3-9ae9-5266e0b9fa63 kind: Event lastTimestamp: "2026-05-20T14:24:09Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T14:24:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:24:09Z" name: error-404-isvc-31b2e-predictor-848cf47897-zpj2x.18b14c250b57ae92 namespace: kserve-ci-e2e-test resourceVersion: "26537" uid: 915ede12-bea9-42a7-96be-f31d49715f1a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:24:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-31b2e-predictor-848cf47897-zpj2x namespace: kserve-ci-e2e-test resourceVersion: "26516" uid: 247807be-896a-45f3-9ae9-5266e0b9fa63 kind: Event lastTimestamp: "2026-05-20T14:24:09Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T14:24:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:24:09Z" name: error-404-isvc-31b2e-predictor-848cf47897-zpj2x.18b14c250bd49b4b namespace: kserve-ci-e2e-test resourceVersion: "26538" uid: a5a92080-6397-4496-9509-7a55282312d3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:24:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-31b2e-predictor-848cf47897-zpj2x namespace: kserve-ci-e2e-test resourceVersion: "26516" uid: 247807be-896a-45f3-9ae9-5266e0b9fa63 kind: Event lastTimestamp: "2026-05-20T14:24:09Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-20T14:24:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:24:09Z" name: error-404-isvc-31b2e-predictor-848cf47897-zpj2x.18b14c250be17234 namespace: kserve-ci-e2e-test resourceVersion: "26539" uid: da7f7658-b3a8-44d9-9d52-619c0b62ed6b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:24:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-31b2e-predictor-848cf47897-zpj2x namespace: kserve-ci-e2e-test resourceVersion: "26516" uid: 247807be-896a-45f3-9ae9-5266e0b9fa63 kind: Event lastTimestamp: "2026-05-20T14:24:09Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T14:24:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:24:09Z" name: error-404-isvc-31b2e-predictor-848cf47897-zpj2x.18b14c250ef52ef7 namespace: kserve-ci-e2e-test resourceVersion: "26542" uid: 4b498326-df50-4b43-8734-8e27b60582ce reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:24:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-31b2e-predictor-848cf47897-zpj2x namespace: kserve-ci-e2e-test resourceVersion: "26516" uid: 247807be-896a-45f3-9ae9-5266e0b9fa63 kind: Event lastTimestamp: "2026-05-20T14:24:09Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:24:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:24:09Z" name: error-404-isvc-31b2e-predictor-848cf47897-zpj2x.18b14c250f71c72d namespace: kserve-ci-e2e-test resourceVersion: "26543" uid: 05d424fa-7da7-40f4-9283-ca7d16f65d6c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-20T14:24:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-31b2e-predictor-848cf47897-zpj2x namespace: kserve-ci-e2e-test resourceVersion: "26516" uid: 247807be-896a-45f3-9ae9-5266e0b9fa63 kind: Event lastTimestamp: "2026-05-20T14:24:47Z" message: 'Readiness probe failed: dial tcp 10.134.0.49:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T14:24:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:24:47Z" name: error-404-isvc-31b2e-predictor-848cf47897-zpj2x.18b14c256c51440c namespace: kserve-ci-e2e-test resourceVersion: "26937" uid: 6f9e5d11-5f3d-4f06-aeb2-313b30677cc5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:33:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-31b2e-predictor-848cf47897-zpj2x namespace: kserve-ci-e2e-test resourceVersion: "26516" uid: 247807be-896a-45f3-9ae9-5266e0b9fa63 kind: Event lastTimestamp: "2026-05-20T14:33:24Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T14:33:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:33:24Z" name: error-404-isvc-31b2e-predictor-848cf47897-zpj2x.18b14ca615bf3086 namespace: kserve-ci-e2e-test resourceVersion: "30606" uid: 22f409c5-b01d-4f53-bd4a-4b9291700299 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:33:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-31b2e-predictor-848cf47897-zpj2x namespace: kserve-ci-e2e-test resourceVersion: "26516" uid: 247807be-896a-45f3-9ae9-5266e0b9fa63 kind: Event lastTimestamp: "2026-05-20T14:33:24Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:33:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:33:24Z" name: error-404-isvc-31b2e-predictor-848cf47897-zpj2x.18b14ca615bf9a0b namespace: kserve-ci-e2e-test resourceVersion: "30607" uid: 996e1440-3ae6-4b12-b35a-4ef6c531361d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:24:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-31b2e-predictor-848cf47897 namespace: kserve-ci-e2e-test resourceVersion: "26504" uid: 2d3179b6-5deb-44dd-9392-d49d7061e79d kind: Event lastTimestamp: "2026-05-20T14:24:09Z" message: 'Created pod: error-404-isvc-31b2e-predictor-848cf47897-zpj2x' metadata: creationTimestamp: "2026-05-20T14:24:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:24:09Z" name: error-404-isvc-31b2e-predictor-848cf47897.18b14c24ebfb0b05 namespace: kserve-ci-e2e-test resourceVersion: "26515" uid: 5ca7bbbc-7c3c-4909-b105-335c138ea471 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:24:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-31b2e-predictor namespace: kserve-ci-e2e-test resourceVersion: "26500" uid: abe66466-771c-4cb6-90ae-fee74f5e87a5 kind: Event lastTimestamp: "2026-05-20T14:24:09Z" message: Scaled up replica set error-404-isvc-31b2e-predictor-848cf47897 from 0 to 1 metadata: creationTimestamp: "2026-05-20T14:24:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:24:09Z" name: error-404-isvc-31b2e-predictor.18b14c24e8fce621 namespace: kserve-ci-e2e-test resourceVersion: "26505" uid: 3e5aac08-6fe9-4337-a503-d2ba50b111ff reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T14:24:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-31b2e-predictor namespace: kserve-ci-e2e-test resourceVersion: "26503" uid: 79f9dbbd-4ab1-45d0-a2d0-57d539a3de76 kind: Event lastTimestamp: "2026-05-20T14:24:39Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:24:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:24:39Z" name: error-404-isvc-31b2e-predictor.18b14c2867b7553f namespace: kserve-ci-e2e-test resourceVersion: "26873" uid: 4d5bf553-6302-44dd-9a4b-238ddcfc1fcb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T14:24:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-31b2e-predictor namespace: kserve-ci-e2e-test resourceVersion: "26503" uid: 79f9dbbd-4ab1-45d0-a2d0-57d539a3de76 kind: Event lastTimestamp: "2026-05-20T14:24:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:24:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:24:39Z" name: error-404-isvc-31b2e-predictor.18b14c28680e0c75 namespace: kserve-ci-e2e-test resourceVersion: "26874" uid: 54ea9944-ba27-4d7e-b51a-12c15550a888 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T14:24:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-31b2e-predictor namespace: kserve-ci-e2e-test resourceVersion: "26699" uid: 79f9dbbd-4ab1-45d0-a2d0-57d539a3de76 kind: Event lastTimestamp: "2026-05-20T14:25:09Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:24:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:25:09Z" name: error-404-isvc-31b2e-predictor.18b14c2f65b4ad02 namespace: kserve-ci-e2e-test resourceVersion: "27152" uid: dd2a8f6b-4ebf-4dc0-958a-3b4d5c110279 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T14:24:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-31b2e-predictor namespace: kserve-ci-e2e-test resourceVersion: "26699" uid: 79f9dbbd-4ab1-45d0-a2d0-57d539a3de76 kind: Event lastTimestamp: "2026-05-20T14:25:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:24:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:25:09Z" name: error-404-isvc-31b2e-predictor.18b14c2f6628cf62 namespace: kserve-ci-e2e-test resourceVersion: "27153" uid: 0dc44e6b-5ee6-43de-b695-b76dcb287d67 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-20T14:24:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-31b2e namespace: kserve-ci-e2e-test resourceVersion: "26480" uid: 666fac47-bf19-4e04-ad79-147a91eb44b4 kind: Event lastTimestamp: "2026-05-20T14:24:57Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T14:24:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:24:57Z" name: error-404-isvc-31b2e.18b14c24e961f999 namespace: kserve-ci-e2e-test resourceVersion: "27028" uid: 4e249c36-4888-4863-adef-3512f8247a4e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:24:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-31b2e namespace: kserve-ci-e2e-test resourceVersion: "26480" uid: 666fac47-bf19-4e04-ad79-147a91eb44b4 kind: Event lastTimestamp: "2026-05-20T14:24:09Z" message: 'Failed to update status for InferenceService "error-404-isvc-31b2e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-31b2e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T14:24:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:24:09Z" name: error-404-isvc-31b2e.18b14c24ea3cf226 namespace: kserve-ci-e2e-test resourceVersion: "26510" uid: f3e55e88-380d-4486-b681-410fdb726a65 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:24:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-31b2e namespace: kserve-ci-e2e-test resourceVersion: "26480" uid: 666fac47-bf19-4e04-ad79-147a91eb44b4 kind: Event lastTimestamp: "2026-05-20T14:24:09Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-31b2e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T14:24:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:24:09Z" name: error-404-isvc-31b2e.18b14c24ea3d5046 namespace: kserve-ci-e2e-test resourceVersion: "26512" uid: 89e35e3c-5b12-4720-b248-7a1d55b1a8cf reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:24:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-31b2e namespace: kserve-ci-e2e-test resourceVersion: "27026" uid: 666fac47-bf19-4e04-ad79-147a91eb44b4 kind: Event lastTimestamp: "2026-05-20T14:24:57Z" message: InferenceService [error-404-isvc-31b2e] is Ready metadata: creationTimestamp: "2026-05-20T14:24:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:24:57Z" name: error-404-isvc-31b2e.18b14c3027d25b7b namespace: kserve-ci-e2e-test resourceVersion: "27027" uid: 2ae69924-96f8-4e97-969f-b05a347e83df reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T14:23:28.818233Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3e303-predictor-7ccc759659-gfmqm namespace: kserve-ci-e2e-test resourceVersion: "25692" uid: 01a6cb98-37b5-49c2-a601-d09952a1df97 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-3e303-predictor-7ccc759659-gfmqm to ip-10-0-136-75.ec2.internal metadata: creationTimestamp: "2026-05-20T14:23:28Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T14:23:28Z" name: error-404-isvc-3e303-predictor-7ccc759659-gfmqm.18b14c1b82141f80 namespace: kserve-ci-e2e-test resourceVersion: "25696" uid: d525a079-d323-416d-9c0a-11e381cfa095 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-747dc7bb84-jx2kx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:23:29Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3e303-predictor-7ccc759659-gfmqm namespace: kserve-ci-e2e-test resourceVersion: "25693" uid: 01a6cb98-37b5-49c2-a601-d09952a1df97 kind: Event lastTimestamp: "2026-05-20T14:23:29Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-3e303-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-20T14:23:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:23:29Z" name: error-404-isvc-3e303-predictor-7ccc759659-gfmqm.18b14c1b8da40c18 namespace: kserve-ci-e2e-test resourceVersion: "25708" uid: fda6a1eb-0e69-4511-bc4d-0bb851d62e52 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:23:29Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3e303-predictor-7ccc759659-gfmqm namespace: kserve-ci-e2e-test resourceVersion: "25700" uid: 01a6cb98-37b5-49c2-a601-d09952a1df97 kind: Event lastTimestamp: "2026-05-20T14:23:29Z" message: Add eth0 [10.134.0.47/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T14:23:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T14:23:29Z" name: error-404-isvc-3e303-predictor-7ccc759659-gfmqm.18b14c1bbfecb036 namespace: kserve-ci-e2e-test resourceVersion: "25732" uid: 8804f24c-ebd0-49f7-b1da-96b47bc4a981 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:23:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3e303-predictor-7ccc759659-gfmqm namespace: kserve-ci-e2e-test resourceVersion: "25693" uid: 01a6cb98-37b5-49c2-a601-d09952a1df97 kind: Event lastTimestamp: "2026-05-20T14:23:29Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1501" already present on machine metadata: creationTimestamp: "2026-05-20T14:23:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:23:29Z" name: error-404-isvc-3e303-predictor-7ccc759659-gfmqm.18b14c1bc199aca3 namespace: kserve-ci-e2e-test resourceVersion: "25734" uid: fcef7560-b517-4593-b9df-f0a6e46bff82 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:23:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3e303-predictor-7ccc759659-gfmqm namespace: kserve-ci-e2e-test resourceVersion: "25693" uid: 01a6cb98-37b5-49c2-a601-d09952a1df97 kind: Event lastTimestamp: "2026-05-20T14:23:29Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T14:23:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:23:29Z" name: error-404-isvc-3e303-predictor-7ccc759659-gfmqm.18b14c1bc5561d15 namespace: kserve-ci-e2e-test resourceVersion: "25735" uid: 47527059-9db9-4d09-91c3-cf442dce9c57 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:23:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3e303-predictor-7ccc759659-gfmqm namespace: kserve-ci-e2e-test resourceVersion: "25693" uid: 01a6cb98-37b5-49c2-a601-d09952a1df97 kind: Event lastTimestamp: "2026-05-20T14:23:29Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T14:23:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:23:29Z" name: error-404-isvc-3e303-predictor-7ccc759659-gfmqm.18b14c1bc5e16f3f namespace: kserve-ci-e2e-test resourceVersion: "25736" uid: 5ce504cb-3f05-42f9-b551-ed53d983e283 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:23:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3e303-predictor-7ccc759659-gfmqm namespace: kserve-ci-e2e-test resourceVersion: "25693" uid: 01a6cb98-37b5-49c2-a601-d09952a1df97 kind: Event lastTimestamp: "2026-05-20T14:23:29Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-20T14:23:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:23:29Z" name: error-404-isvc-3e303-predictor-7ccc759659-gfmqm.18b14c1bc5eed307 namespace: kserve-ci-e2e-test resourceVersion: "25737" uid: 5d973575-550d-4d0b-9d30-9070f5004844 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:23:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3e303-predictor-7ccc759659-gfmqm namespace: kserve-ci-e2e-test resourceVersion: "25693" uid: 01a6cb98-37b5-49c2-a601-d09952a1df97 kind: Event lastTimestamp: "2026-05-20T14:23:30Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T14:23:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:23:30Z" name: error-404-isvc-3e303-predictor-7ccc759659-gfmqm.18b14c1bc9069d1e namespace: kserve-ci-e2e-test resourceVersion: "25738" uid: ebe36807-66d5-4642-af16-762ce177bff6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:23:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3e303-predictor-7ccc759659-gfmqm namespace: kserve-ci-e2e-test resourceVersion: "25693" uid: 01a6cb98-37b5-49c2-a601-d09952a1df97 kind: Event lastTimestamp: "2026-05-20T14:23:30Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:23:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:23:30Z" name: error-404-isvc-3e303-predictor-7ccc759659-gfmqm.18b14c1bc9845020 namespace: kserve-ci-e2e-test resourceVersion: "25740" uid: ec868a74-0094-4a4b-b4ad-03c0990dd573 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-20T14:23:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3e303-predictor-7ccc759659-gfmqm namespace: kserve-ci-e2e-test resourceVersion: "25693" uid: 01a6cb98-37b5-49c2-a601-d09952a1df97 kind: Event lastTimestamp: "2026-05-20T14:24:07Z" message: 'Readiness probe failed: dial tcp 10.134.0.47:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T14:23:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:24:07Z" name: error-404-isvc-3e303-predictor-7ccc759659-gfmqm.18b14c1c11e35879 namespace: kserve-ci-e2e-test resourceVersion: "26403" uid: 5caeb5b7-08bb-48cc-aaeb-7cddfbe8fb11 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:40:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3e303-predictor-7ccc759659-gfmqm namespace: kserve-ci-e2e-test resourceVersion: "25693" uid: 01a6cb98-37b5-49c2-a601-d09952a1df97 kind: Event lastTimestamp: "2026-05-20T14:40:48Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T14:40:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:40:48Z" name: error-404-isvc-3e303-predictor-7ccc759659-gfmqm.18b14d0d7efd35f2 namespace: kserve-ci-e2e-test resourceVersion: "33674" uid: ecd977b4-9428-4400-9c00-6f63d8174f81 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:40:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3e303-predictor-7ccc759659-gfmqm namespace: kserve-ci-e2e-test resourceVersion: "25693" uid: 01a6cb98-37b5-49c2-a601-d09952a1df97 kind: Event lastTimestamp: "2026-05-20T14:40:48Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:40:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:40:48Z" name: error-404-isvc-3e303-predictor-7ccc759659-gfmqm.18b14d0d7efd92fe namespace: kserve-ci-e2e-test resourceVersion: "33675" uid: 19ce7d75-db21-45e3-81f9-b6c202fbc5a4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:23:28Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-3e303-predictor-7ccc759659 namespace: kserve-ci-e2e-test resourceVersion: "25682" uid: 6a6c48f5-3dea-42c8-83c0-590fa2bfa654 kind: Event lastTimestamp: "2026-05-20T14:23:28Z" message: 'Created pod: error-404-isvc-3e303-predictor-7ccc759659-gfmqm' metadata: creationTimestamp: "2026-05-20T14:23:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:23:28Z" name: error-404-isvc-3e303-predictor-7ccc759659.18b14c1b81b71fdf namespace: kserve-ci-e2e-test resourceVersion: "25695" uid: 2aa1ec43-dd71-4340-8f43-9ba3369f35e7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:23:28Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-3e303-predictor namespace: kserve-ci-e2e-test resourceVersion: "25680" uid: 0d2dd92f-de8e-4161-8156-d7c6b611fba2 kind: Event lastTimestamp: "2026-05-20T14:23:28Z" message: Scaled up replica set error-404-isvc-3e303-predictor-7ccc759659 from 0 to 1 metadata: creationTimestamp: "2026-05-20T14:23:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:23:28Z" name: error-404-isvc-3e303-predictor.18b14c1b7eb6f77b namespace: kserve-ci-e2e-test resourceVersion: "25684" uid: e2411d22-9f55-4f7d-9085-2afb879e06ab reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:23:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3e303-predictor namespace: kserve-ci-e2e-test resourceVersion: "25681" uid: 6d8d48f7-c951-4361-96b7-bfbe7709e303 kind: Event lastTimestamp: "2026-05-20T14:23:43Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:23:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:23:43Z" name: error-404-isvc-3e303-predictor.18b14c1efd42d14b namespace: kserve-ci-e2e-test resourceVersion: "26072" uid: 1b77038c-32f4-477d-8a35-2e43d5d0e756 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:23:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3e303-predictor namespace: kserve-ci-e2e-test resourceVersion: "25681" uid: 6d8d48f7-c951-4361-96b7-bfbe7709e303 kind: Event lastTimestamp: "2026-05-20T14:23:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:23:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:23:43Z" name: error-404-isvc-3e303-predictor.18b14c1efdc0a350 namespace: kserve-ci-e2e-test resourceVersion: "26074" uid: 9a2f86b5-c7f9-487f-ad7e-32d907f8b36d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-20T14:23:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3e303-predictor namespace: kserve-ci-e2e-test resourceVersion: "26073" uid: 6d8d48f7-c951-4361-96b7-bfbe7709e303 kind: Event lastTimestamp: "2026-05-20T14:24:58Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:23:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:24:58Z" name: error-404-isvc-3e303-predictor.18b14c227c7f5922 namespace: kserve-ci-e2e-test resourceVersion: "27050" uid: 6a46ba5a-2394-4a15-9166-105c3aa482e5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-20T14:23:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3e303-predictor namespace: kserve-ci-e2e-test resourceVersion: "26073" uid: 6d8d48f7-c951-4361-96b7-bfbe7709e303 kind: Event lastTimestamp: "2026-05-20T14:24:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:23:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:24:58Z" name: error-404-isvc-3e303-predictor.18b14c227cd77bd5 namespace: kserve-ci-e2e-test resourceVersion: "27051" uid: 19782db0-339f-4f9b-9407-d0c939fab83b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-20T14:23:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3e303 namespace: kserve-ci-e2e-test resourceVersion: "25667" uid: fa89ee2e-a88c-41e6-a810-a9c4fe37c418 kind: Event lastTimestamp: "2026-05-20T14:24:17Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T14:23:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:24:17Z" name: error-404-isvc-3e303.18b14c1b7e81f994 namespace: kserve-ci-e2e-test resourceVersion: "26648" uid: d7c2e909-648e-47fe-b833-47d1d1b6e5f1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:23:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3e303 namespace: kserve-ci-e2e-test resourceVersion: "25667" uid: fa89ee2e-a88c-41e6-a810-a9c4fe37c418 kind: Event lastTimestamp: "2026-05-20T14:23:28Z" message: 'Failed to update status for InferenceService "error-404-isvc-3e303": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-3e303": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T14:23:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:23:28Z" name: error-404-isvc-3e303.18b14c1b7f0b80f2 namespace: kserve-ci-e2e-test resourceVersion: "25686" uid: 0d2e6973-dd58-45e3-9d80-cfa4ca344f3d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:23:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3e303 namespace: kserve-ci-e2e-test resourceVersion: "25667" uid: fa89ee2e-a88c-41e6-a810-a9c4fe37c418 kind: Event lastTimestamp: "2026-05-20T14:23:28Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-3e303": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T14:23:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:23:28Z" name: error-404-isvc-3e303.18b14c1b7f0bcbf7 namespace: kserve-ci-e2e-test resourceVersion: "25688" uid: ffabd425-aaa4-4620-b01b-8ad91db6517e reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:24:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3e303 namespace: kserve-ci-e2e-test resourceVersion: "26646" uid: fa89ee2e-a88c-41e6-a810-a9c4fe37c418 kind: Event lastTimestamp: "2026-05-20T14:24:17Z" message: InferenceService [error-404-isvc-3e303] is Ready metadata: creationTimestamp: "2026-05-20T14:24:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:24:17Z" name: error-404-isvc-3e303.18b14c26ce570767 namespace: kserve-ci-e2e-test resourceVersion: "26647" uid: 2d05b9eb-396a-4ae2-a81d-8a00aa4fde1f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T14:01:38.496639Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-69925-predictor-5d57446965-8zjs4 namespace: kserve-ci-e2e-test resourceVersion: "14568" uid: 3b977c2d-8ee4-4db5-bb33-e828b9e1db25 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-69925-predictor-5d57446965-8zjs4 to ip-10-0-136-75.ec2.internal metadata: creationTimestamp: "2026-05-20T14:01:38Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T14:01:38Z" name: error-404-isvc-69925-predictor-5d57446965-8zjs4.18b14aea6cd358a9 namespace: kserve-ci-e2e-test resourceVersion: "14574" uid: 9a1aa65f-4f51-4229-944f-fdd4b20a06ca reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-747dc7bb84-jx2kx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:38Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-69925-predictor-5d57446965-8zjs4 namespace: kserve-ci-e2e-test resourceVersion: "14581" uid: 3b977c2d-8ee4-4db5-bb33-e828b9e1db25 kind: Event lastTimestamp: "2026-05-20T14:01:38Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T14:01:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T14:01:38Z" name: error-404-isvc-69925-predictor-5d57446965-8zjs4.18b14aea8714a76b namespace: kserve-ci-e2e-test resourceVersion: "14615" uid: 5a8e2c8e-46a6-4781-9c70-362583562af7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-69925-predictor-5d57446965-8zjs4 namespace: kserve-ci-e2e-test resourceVersion: "14570" uid: 3b977c2d-8ee4-4db5-bb33-e828b9e1db25 kind: Event lastTimestamp: "2026-05-20T14:01:38Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1501" metadata: creationTimestamp: "2026-05-20T14:01:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:38Z" name: error-404-isvc-69925-predictor-5d57446965-8zjs4.18b14aea88de370f namespace: kserve-ci-e2e-test resourceVersion: "14624" uid: dc5fb3f4-78f1-4b5b-91ee-dda1ba763c32 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-69925-predictor-5d57446965-8zjs4 namespace: kserve-ci-e2e-test resourceVersion: "14570" uid: 3b977c2d-8ee4-4db5-bb33-e828b9e1db25 kind: Event lastTimestamp: "2026-05-20T14:01:50Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1501" in 11.315s (11.315s including waiting). Image size: 1334723672 bytes.' metadata: creationTimestamp: "2026-05-20T14:01:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:50Z" name: error-404-isvc-69925-predictor-5d57446965-8zjs4.18b14aed2b58b12e namespace: kserve-ci-e2e-test resourceVersion: "14772" uid: 50d4880b-668d-4f5d-a20d-ff1ba24be914 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-69925-predictor-5d57446965-8zjs4 namespace: kserve-ci-e2e-test resourceVersion: "14570" uid: 3b977c2d-8ee4-4db5-bb33-e828b9e1db25 kind: Event lastTimestamp: "2026-05-20T14:01:50Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T14:01:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:50Z" name: error-404-isvc-69925-predictor-5d57446965-8zjs4.18b14aed2f54aeff namespace: kserve-ci-e2e-test resourceVersion: "14773" uid: e708fe5c-b1e6-4127-afbf-788895b48835 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-69925-predictor-5d57446965-8zjs4 namespace: kserve-ci-e2e-test resourceVersion: "14570" uid: 3b977c2d-8ee4-4db5-bb33-e828b9e1db25 kind: Event lastTimestamp: "2026-05-20T14:01:50Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T14:01:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:50Z" name: error-404-isvc-69925-predictor-5d57446965-8zjs4.18b14aed2ffcfc65 namespace: kserve-ci-e2e-test resourceVersion: "14774" uid: 2b74e9fd-9dd9-491a-bf70-a2893a0df459 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-69925-predictor-5d57446965-8zjs4 namespace: kserve-ci-e2e-test resourceVersion: "14570" uid: 3b977c2d-8ee4-4db5-bb33-e828b9e1db25 kind: Event lastTimestamp: "2026-05-20T14:01:50Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-05-20T14:01:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:50Z" name: error-404-isvc-69925-predictor-5d57446965-8zjs4.18b14aed300a6c19 namespace: kserve-ci-e2e-test resourceVersion: "14775" uid: 09c89dec-8c37-498a-92ba-d68877917f45 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-69925-predictor-5d57446965-8zjs4 namespace: kserve-ci-e2e-test resourceVersion: "14570" uid: 3b977c2d-8ee4-4db5-bb33-e828b9e1db25 kind: Event lastTimestamp: "2026-05-20T14:01:52Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.179s (2.179s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-05-20T14:01:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:52Z" name: error-404-isvc-69925-predictor-5d57446965-8zjs4.18b14aedb1ed17b1 namespace: kserve-ci-e2e-test resourceVersion: "14794" uid: ed726b24-9f12-4086-b711-24641e80d20c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-69925-predictor-5d57446965-8zjs4 namespace: kserve-ci-e2e-test resourceVersion: "14570" uid: 3b977c2d-8ee4-4db5-bb33-e828b9e1db25 kind: Event lastTimestamp: "2026-05-20T14:01:52Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T14:01:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:52Z" name: error-404-isvc-69925-predictor-5d57446965-8zjs4.18b14aedb5014c6a namespace: kserve-ci-e2e-test resourceVersion: "14795" uid: 0ac8f2b5-8503-493a-b6c3-862afbca6a2f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-69925-predictor-5d57446965-8zjs4 namespace: kserve-ci-e2e-test resourceVersion: "14570" uid: 3b977c2d-8ee4-4db5-bb33-e828b9e1db25 kind: Event lastTimestamp: "2026-05-20T14:01:52Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:01:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:52Z" name: error-404-isvc-69925-predictor-5d57446965-8zjs4.18b14aedb572a0de namespace: kserve-ci-e2e-test resourceVersion: "14796" uid: f90710da-ee0d-40cd-9f1a-d65c86a38795 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-20T14:01:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-69925-predictor-5d57446965-8zjs4 namespace: kserve-ci-e2e-test resourceVersion: "14570" uid: 3b977c2d-8ee4-4db5-bb33-e828b9e1db25 kind: Event lastTimestamp: "2026-05-20T14:02:28Z" message: 'Readiness probe failed: dial tcp 10.134.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T14:01:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:02:28Z" name: error-404-isvc-69925-predictor-5d57446965-8zjs4.18b14aedb90255da namespace: kserve-ci-e2e-test resourceVersion: "15142" uid: e55fbe33-4b28-492f-b342-a944137b93d2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-69925-predictor-5d57446965-8zjs4 namespace: kserve-ci-e2e-test resourceVersion: "14570" uid: 3b977c2d-8ee4-4db5-bb33-e828b9e1db25 kind: Event lastTimestamp: "2026-05-20T14:03:18Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T14:03:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:18Z" name: error-404-isvc-69925-predictor-5d57446965-8zjs4.18b14b01bd555e54 namespace: kserve-ci-e2e-test resourceVersion: "15689" uid: 486b050d-8f01-40b4-bf67-9f0c1dd341b3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-69925-predictor-5d57446965-8zjs4 namespace: kserve-ci-e2e-test resourceVersion: "14570" uid: 3b977c2d-8ee4-4db5-bb33-e828b9e1db25 kind: Event lastTimestamp: "2026-05-20T14:03:18Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:03:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:18Z" name: error-404-isvc-69925-predictor-5d57446965-8zjs4.18b14b01bd564dab namespace: kserve-ci-e2e-test resourceVersion: "15693" uid: 067361ed-52e4-4dfd-a827-ad902beb0651 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-69925-predictor-5d57446965-8zjs4 namespace: kserve-ci-e2e-test resourceVersion: "14570" uid: 3b977c2d-8ee4-4db5-bb33-e828b9e1db25 kind: Event lastTimestamp: "2026-05-20T14:03:18Z" message: 'Readiness probe failed: Get "https://10.134.0.31:8643/healthz": dial tcp 10.134.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-20T14:03:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:18Z" name: error-404-isvc-69925-predictor-5d57446965-8zjs4.18b14b01bf2bcbdd namespace: kserve-ci-e2e-test resourceVersion: "15699" uid: d0e9e40a-4219-4055-95d7-cb709a8cc1ac reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:38Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-69925-predictor-5d57446965 namespace: kserve-ci-e2e-test resourceVersion: "14560" uid: 64f08a47-d354-4c34-8eb5-027bfbf40596 kind: Event lastTimestamp: "2026-05-20T14:01:38Z" message: 'Created pod: error-404-isvc-69925-predictor-5d57446965-8zjs4' metadata: creationTimestamp: "2026-05-20T14:01:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:01:38Z" name: error-404-isvc-69925-predictor-5d57446965.18b14aea6c5194c4 namespace: kserve-ci-e2e-test resourceVersion: "14571" uid: c4eb348f-1b24-4823-9b72-399b568bf02b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:38Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-69925-predictor namespace: kserve-ci-e2e-test resourceVersion: "14557" uid: dd44176d-d3c9-4e57-a6a6-232a8967c31c kind: Event lastTimestamp: "2026-05-20T14:01:38Z" message: Scaled up replica set error-404-isvc-69925-predictor-5d57446965 from 0 to 1 metadata: creationTimestamp: "2026-05-20T14:01:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:01:38Z" name: error-404-isvc-69925-predictor.18b14aea675549da namespace: kserve-ci-e2e-test resourceVersion: "14563" uid: 54f6d343-758c-41e5-83f7-83d54b656d09 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T14:01:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-69925-predictor namespace: kserve-ci-e2e-test resourceVersion: "14558" uid: b43c2778-b8be-4c22-a9ef-94309ceb5753 kind: Event lastTimestamp: "2026-05-20T14:02:23Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:01:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:02:23Z" name: error-404-isvc-69925-predictor.18b14aede5a6a289 namespace: kserve-ci-e2e-test resourceVersion: "15092" uid: 82ddbf03-c7e0-46ba-a0c3-d2fed70c8fda reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T14:01:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-69925-predictor namespace: kserve-ci-e2e-test resourceVersion: "14558" uid: b43c2778-b8be-4c22-a9ef-94309ceb5753 kind: Event lastTimestamp: "2026-05-20T14:02:23Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:01:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:02:23Z" name: error-404-isvc-69925-predictor.18b14aede657e9bd namespace: kserve-ci-e2e-test resourceVersion: "15093" uid: 65043a3d-8d2d-4e48-acc9-a4979bd1679e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T14:02:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-69925-predictor namespace: kserve-ci-e2e-test resourceVersion: "14811" uid: b43c2778-b8be-4c22-a9ef-94309ceb5753 kind: Event lastTimestamp: "2026-05-20T14:03:08Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:02:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:03:08Z" name: error-404-isvc-69925-predictor.18b14af863cac28c namespace: kserve-ci-e2e-test resourceVersion: "15496" uid: 21c7b193-ec53-444c-939c-8337324d48f7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T14:02:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-69925-predictor namespace: kserve-ci-e2e-test resourceVersion: "14811" uid: b43c2778-b8be-4c22-a9ef-94309ceb5753 kind: Event lastTimestamp: "2026-05-20T14:03:08Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:02:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:03:08Z" name: error-404-isvc-69925-predictor.18b14af8644fdef7 namespace: kserve-ci-e2e-test resourceVersion: "15497" uid: 9bc6d72a-139d-4743-a129-ea0ad1751dac reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-20T14:01:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-69925 namespace: kserve-ci-e2e-test resourceVersion: "14539" uid: abf677dd-0d2a-42e8-a4f5-6e3f719f92cf kind: Event lastTimestamp: "2026-05-20T14:02:38Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T14:01:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:02:38Z" name: error-404-isvc-69925.18b14aea66bdca7c namespace: kserve-ci-e2e-test resourceVersion: "15231" uid: 46931acc-47de-4233-8453-e220acf94c3d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:02:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-69925 namespace: kserve-ci-e2e-test resourceVersion: "15229" uid: abf677dd-0d2a-42e8-a4f5-6e3f719f92cf kind: Event lastTimestamp: "2026-05-20T14:02:38Z" message: InferenceService [error-404-isvc-69925] is Ready metadata: creationTimestamp: "2026-05-20T14:02:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:02:38Z" name: error-404-isvc-69925.18b14af8758737b6 namespace: kserve-ci-e2e-test resourceVersion: "15230" uid: 5c6b8e8f-20b3-4495-ab35-d9507ceef682 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T14:14:13.808403Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-734fc-predictor-5995d7675c-86m5w namespace: kserve-ci-e2e-test resourceVersion: "21390" uid: 7bb62959-33c7-4cf3-824a-616ee8f239e2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-734fc-predictor-5995d7675c-86m5w to ip-10-0-136-75.ec2.internal metadata: creationTimestamp: "2026-05-20T14:14:13Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T14:14:13Z" name: error-404-isvc-734fc-predictor-5995d7675c-86m5w.18b14b9a48ea82e1 namespace: kserve-ci-e2e-test resourceVersion: "21396" uid: e1b765e4-9ab8-42cd-9ab0-8a8ff1b2e761 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-747dc7bb84-jx2kx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:14:13Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-734fc-predictor-5995d7675c-86m5w namespace: kserve-ci-e2e-test resourceVersion: "21393" uid: 7bb62959-33c7-4cf3-824a-616ee8f239e2 kind: Event lastTimestamp: "2026-05-20T14:14:13Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-734fc-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-20T14:14:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:14:14Z" name: error-404-isvc-734fc-predictor-5995d7675c-86m5w.18b14b9a54528d70 namespace: kserve-ci-e2e-test resourceVersion: "21411" uid: 7de7b52e-b7d2-4433-9370-976ddc56a3ae reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:14:14Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-734fc-predictor-5995d7675c-86m5w namespace: kserve-ci-e2e-test resourceVersion: "21403" uid: 7bb62959-33c7-4cf3-824a-616ee8f239e2 kind: Event lastTimestamp: "2026-05-20T14:14:14Z" message: Add eth0 [10.134.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T14:14:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T14:14:14Z" name: error-404-isvc-734fc-predictor-5995d7675c-86m5w.18b14b9a87fa8a2d namespace: kserve-ci-e2e-test resourceVersion: "21437" uid: 0d66fd43-3072-4e96-9d77-dee00c8d7b86 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:14:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-734fc-predictor-5995d7675c-86m5w namespace: kserve-ci-e2e-test resourceVersion: "21393" uid: 7bb62959-33c7-4cf3-824a-616ee8f239e2 kind: Event lastTimestamp: "2026-05-20T14:14:14Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1501" already present on machine metadata: creationTimestamp: "2026-05-20T14:14:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:14:14Z" name: error-404-isvc-734fc-predictor-5995d7675c-86m5w.18b14b9a8975308a namespace: kserve-ci-e2e-test resourceVersion: "21439" uid: 14030aef-6a98-463f-a713-c22a878ed5f1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:14:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-734fc-predictor-5995d7675c-86m5w namespace: kserve-ci-e2e-test resourceVersion: "21393" uid: 7bb62959-33c7-4cf3-824a-616ee8f239e2 kind: Event lastTimestamp: "2026-05-20T14:14:14Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T14:14:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:14:14Z" name: error-404-isvc-734fc-predictor-5995d7675c-86m5w.18b14b9a8dac88ac namespace: kserve-ci-e2e-test resourceVersion: "21440" uid: 3753b48f-160a-4de3-8ac4-7aa49cbfb999 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:14:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-734fc-predictor-5995d7675c-86m5w namespace: kserve-ci-e2e-test resourceVersion: "21393" uid: 7bb62959-33c7-4cf3-824a-616ee8f239e2 kind: Event lastTimestamp: "2026-05-20T14:14:14Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T14:14:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:14:14Z" name: error-404-isvc-734fc-predictor-5995d7675c-86m5w.18b14b9a8e2c367b namespace: kserve-ci-e2e-test resourceVersion: "21441" uid: 624773b1-1b4f-41c7-9bed-e63ec70b2f65 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:14:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-734fc-predictor-5995d7675c-86m5w namespace: kserve-ci-e2e-test resourceVersion: "21393" uid: 7bb62959-33c7-4cf3-824a-616ee8f239e2 kind: Event lastTimestamp: "2026-05-20T14:14:14Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-20T14:14:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:14:14Z" name: error-404-isvc-734fc-predictor-5995d7675c-86m5w.18b14b9a8e39e46d namespace: kserve-ci-e2e-test resourceVersion: "21442" uid: a9dec9bb-9baf-40e4-b61d-8c163f12d319 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:14:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-734fc-predictor-5995d7675c-86m5w namespace: kserve-ci-e2e-test resourceVersion: "21393" uid: 7bb62959-33c7-4cf3-824a-616ee8f239e2 kind: Event lastTimestamp: "2026-05-20T14:14:15Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T14:14:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:14:15Z" name: error-404-isvc-734fc-predictor-5995d7675c-86m5w.18b14b9a922d3b8b namespace: kserve-ci-e2e-test resourceVersion: "21446" uid: 4a54154b-9d6a-4811-97d9-ad0a2dfe2442 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:14:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-734fc-predictor-5995d7675c-86m5w namespace: kserve-ci-e2e-test resourceVersion: "21393" uid: 7bb62959-33c7-4cf3-824a-616ee8f239e2 kind: Event lastTimestamp: "2026-05-20T14:14:15Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:14:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:14:15Z" name: error-404-isvc-734fc-predictor-5995d7675c-86m5w.18b14b9a92ad73f8 namespace: kserve-ci-e2e-test resourceVersion: "21448" uid: 1d1d677d-5c7a-4d6a-b75f-9b0195e2184c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-20T14:14:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-734fc-predictor-5995d7675c-86m5w namespace: kserve-ci-e2e-test resourceVersion: "21393" uid: 7bb62959-33c7-4cf3-824a-616ee8f239e2 kind: Event lastTimestamp: "2026-05-20T14:14:52Z" message: 'Readiness probe failed: dial tcp 10.134.0.43:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T14:14:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:14:52Z" name: error-404-isvc-734fc-predictor-5995d7675c-86m5w.18b14b9addae7a6e namespace: kserve-ci-e2e-test resourceVersion: "21815" uid: a1f9002f-f475-4d97-9340-d3f257f6a2e6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:23:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-734fc-predictor-5995d7675c-86m5w namespace: kserve-ci-e2e-test resourceVersion: "21393" uid: 7bb62959-33c7-4cf3-824a-616ee8f239e2 kind: Event lastTimestamp: "2026-05-20T14:23:28Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T14:23:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:23:28Z" name: error-404-isvc-734fc-predictor-5995d7675c-86m5w.18b14c1b6fea7dc6 namespace: kserve-ci-e2e-test resourceVersion: "25627" uid: c1b16d6a-ec3f-45b8-894b-a6cbea852e5f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:23:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-734fc-predictor-5995d7675c-86m5w namespace: kserve-ci-e2e-test resourceVersion: "21393" uid: 7bb62959-33c7-4cf3-824a-616ee8f239e2 kind: Event lastTimestamp: "2026-05-20T14:23:28Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:23:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:23:28Z" name: error-404-isvc-734fc-predictor-5995d7675c-86m5w.18b14c1b6fedc612 namespace: kserve-ci-e2e-test resourceVersion: "25629" uid: f5cef02a-7457-4a45-8636-3adf74e38763 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:14:13Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-734fc-predictor-5995d7675c namespace: kserve-ci-e2e-test resourceVersion: "21384" uid: 3724fea8-0977-438d-a361-5102a7373af0 kind: Event lastTimestamp: "2026-05-20T14:14:13Z" message: 'Created pod: error-404-isvc-734fc-predictor-5995d7675c-86m5w' metadata: creationTimestamp: "2026-05-20T14:14:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:14:13Z" name: error-404-isvc-734fc-predictor-5995d7675c.18b14b9a48c2b0b9 namespace: kserve-ci-e2e-test resourceVersion: "21394" uid: 23cb5b01-1dc0-45a0-9751-d564c6fa5962 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:14:13Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-734fc-predictor namespace: kserve-ci-e2e-test resourceVersion: "21382" uid: bc1619db-5da3-418c-a9ed-8c1495e3f166 kind: Event lastTimestamp: "2026-05-20T14:14:13Z" message: Scaled up replica set error-404-isvc-734fc-predictor-5995d7675c from 0 to 1 metadata: creationTimestamp: "2026-05-20T14:14:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:14:13Z" name: error-404-isvc-734fc-predictor.18b14b9a46ae76b0 namespace: kserve-ci-e2e-test resourceVersion: "21386" uid: 3fa4e94f-2abf-4cbb-b9fc-b69776381399 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:14:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-734fc-predictor namespace: kserve-ci-e2e-test resourceVersion: "21383" uid: 10778569-87ab-450c-bc7a-6e3aa4ef9d4d kind: Event lastTimestamp: "2026-05-20T14:14:28Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:14:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:14:28Z" name: error-404-isvc-734fc-predictor.18b14b9dc53f3647 namespace: kserve-ci-e2e-test resourceVersion: "21564" uid: c0363650-0030-48e3-a20c-b6a0a61fc8d2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:14:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-734fc-predictor namespace: kserve-ci-e2e-test resourceVersion: "21383" uid: 10778569-87ab-450c-bc7a-6e3aa4ef9d4d kind: Event lastTimestamp: "2026-05-20T14:14:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:14:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:14:28Z" name: error-404-isvc-734fc-predictor.18b14b9dc5b60c7c namespace: kserve-ci-e2e-test resourceVersion: "21566" uid: 57c64a0b-242f-4e55-a1f2-964376096c87 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-20T14:14:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-734fc-predictor namespace: kserve-ci-e2e-test resourceVersion: "21565" uid: 10778569-87ab-450c-bc7a-6e3aa4ef9d4d kind: Event lastTimestamp: "2026-05-20T14:15:28Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:14:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:15:28Z" name: error-404-isvc-734fc-predictor.18b14ba1447d13f2 namespace: kserve-ci-e2e-test resourceVersion: "22163" uid: 5a43e66f-4531-4233-94d2-2a29018a1dee reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-20T14:14:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-734fc-predictor namespace: kserve-ci-e2e-test resourceVersion: "21565" uid: 10778569-87ab-450c-bc7a-6e3aa4ef9d4d kind: Event lastTimestamp: "2026-05-20T14:15:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:14:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:15:28Z" name: error-404-isvc-734fc-predictor.18b14ba144f4d76c namespace: kserve-ci-e2e-test resourceVersion: "22164" uid: 65c100d3-2d6a-4ee6-a72d-b8eee27e7320 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-20T14:14:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-734fc namespace: kserve-ci-e2e-test resourceVersion: "21352" uid: 7de4a913-1f6f-4e88-91dd-ac5c3b5dac4e kind: Event lastTimestamp: "2026-05-20T14:15:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T14:14:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:15:02Z" name: error-404-isvc-734fc.18b14b9a46721c61 namespace: kserve-ci-e2e-test resourceVersion: "21917" uid: d596ee9d-f061-4aba-9553-3b653e8d05f0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:14:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-734fc namespace: kserve-ci-e2e-test resourceVersion: "21352" uid: 7de4a913-1f6f-4e88-91dd-ac5c3b5dac4e kind: Event lastTimestamp: "2026-05-20T14:14:13Z" message: 'Failed to update status for InferenceService "error-404-isvc-734fc": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-734fc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T14:14:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:14:13Z" name: error-404-isvc-734fc.18b14b9a46e0c599 namespace: kserve-ci-e2e-test resourceVersion: "21388" uid: 5dd4fa9a-3fa9-456e-92d2-f4b204ea76d5 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:14:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-734fc namespace: kserve-ci-e2e-test resourceVersion: "21352" uid: 7de4a913-1f6f-4e88-91dd-ac5c3b5dac4e kind: Event lastTimestamp: "2026-05-20T14:14:13Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-734fc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T14:14:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:14:13Z" name: error-404-isvc-734fc.18b14b9a46e1270b namespace: kserve-ci-e2e-test resourceVersion: "21389" uid: ea9d053e-e784-413a-94f2-db33c0715f25 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:15:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-734fc namespace: kserve-ci-e2e-test resourceVersion: "21915" uid: 7de4a913-1f6f-4e88-91dd-ac5c3b5dac4e kind: Event lastTimestamp: "2026-05-20T14:15:02Z" message: InferenceService [error-404-isvc-734fc] is Ready metadata: creationTimestamp: "2026-05-20T14:15:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:15:02Z" name: error-404-isvc-734fc.18b14ba59b2cdae5 namespace: kserve-ci-e2e-test resourceVersion: "21916" uid: 692795cf-a340-4d9a-9850-2e74ec8d8379 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T14:03:18.687238Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-83116-predictor-849469f6df-2lzrf namespace: kserve-ci-e2e-test resourceVersion: "15701" uid: c5909edf-61bd-432d-bd91-b5e6418f4e8d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-83116-predictor-849469f6df-2lzrf to ip-10-0-136-75.ec2.internal metadata: creationTimestamp: "2026-05-20T14:03:18Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T14:03:18Z" name: error-404-isvc-83116-predictor-849469f6df-2lzrf.18b14b01c0a684d0 namespace: kserve-ci-e2e-test resourceVersion: "15705" uid: 2c6cfc6a-0234-4d50-9d33-b2d6b235dc3c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-747dc7bb84-jx2kx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:19Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-83116-predictor-849469f6df-2lzrf namespace: kserve-ci-e2e-test resourceVersion: "15710" uid: c5909edf-61bd-432d-bd91-b5e6418f4e8d kind: Event lastTimestamp: "2026-05-20T14:03:19Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T14:03:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T14:03:19Z" name: error-404-isvc-83116-predictor-849469f6df-2lzrf.18b14b01da5c59cb namespace: kserve-ci-e2e-test resourceVersion: "15723" uid: a04e4fb9-bad6-4008-bb32-8e4db271bba9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-83116-predictor-849469f6df-2lzrf namespace: kserve-ci-e2e-test resourceVersion: "15702" uid: c5909edf-61bd-432d-bd91-b5e6418f4e8d kind: Event lastTimestamp: "2026-05-20T14:03:19Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1501" already present on machine metadata: creationTimestamp: "2026-05-20T14:03:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:19Z" name: error-404-isvc-83116-predictor-849469f6df-2lzrf.18b14b01dc4a295d namespace: kserve-ci-e2e-test resourceVersion: "15729" uid: d904f989-b07a-4377-83ec-b0e6cabec243 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-83116-predictor-849469f6df-2lzrf namespace: kserve-ci-e2e-test resourceVersion: "15702" uid: c5909edf-61bd-432d-bd91-b5e6418f4e8d kind: Event lastTimestamp: "2026-05-20T14:03:19Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T14:03:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:19Z" name: error-404-isvc-83116-predictor-849469f6df-2lzrf.18b14b01df53b6f3 namespace: kserve-ci-e2e-test resourceVersion: "15730" uid: e929774e-e7db-45d1-809e-e89438b8a965 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-83116-predictor-849469f6df-2lzrf namespace: kserve-ci-e2e-test resourceVersion: "15702" uid: c5909edf-61bd-432d-bd91-b5e6418f4e8d kind: Event lastTimestamp: "2026-05-20T14:03:19Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T14:03:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:19Z" name: error-404-isvc-83116-predictor-849469f6df-2lzrf.18b14b01dfbdf418 namespace: kserve-ci-e2e-test resourceVersion: "15731" uid: 5b6025c2-8a6c-425f-b8f5-7309009568e1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-83116-predictor-849469f6df-2lzrf namespace: kserve-ci-e2e-test resourceVersion: "15702" uid: c5909edf-61bd-432d-bd91-b5e6418f4e8d kind: Event lastTimestamp: "2026-05-20T14:03:19Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-20T14:03:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:19Z" name: error-404-isvc-83116-predictor-849469f6df-2lzrf.18b14b01dfcb3956 namespace: kserve-ci-e2e-test resourceVersion: "15732" uid: c772947d-2923-4900-b385-b1b037ca22e3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-83116-predictor-849469f6df-2lzrf namespace: kserve-ci-e2e-test resourceVersion: "15702" uid: c5909edf-61bd-432d-bd91-b5e6418f4e8d kind: Event lastTimestamp: "2026-05-20T14:03:19Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T14:03:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:19Z" name: error-404-isvc-83116-predictor-849469f6df-2lzrf.18b14b01e3c8d953 namespace: kserve-ci-e2e-test resourceVersion: "15733" uid: f8b7b115-2b81-4ec7-88a8-0ad42c98e518 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-83116-predictor-849469f6df-2lzrf namespace: kserve-ci-e2e-test resourceVersion: "15702" uid: c5909edf-61bd-432d-bd91-b5e6418f4e8d kind: Event lastTimestamp: "2026-05-20T14:03:19Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:03:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:19Z" name: error-404-isvc-83116-predictor-849469f6df-2lzrf.18b14b01e43a1df9 namespace: kserve-ci-e2e-test resourceVersion: "15734" uid: 47cd01c0-c50b-4c95-85c2-faf13cc647eb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-05-20T14:03:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-83116-predictor-849469f6df-2lzrf namespace: kserve-ci-e2e-test resourceVersion: "15702" uid: c5909edf-61bd-432d-bd91-b5e6418f4e8d kind: Event lastTimestamp: "2026-05-20T14:12:36Z" message: 'Readiness probe failed: dial tcp 10.134.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T14:03:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:12:36Z" name: error-404-isvc-83116-predictor-849469f6df-2lzrf.18b14b0248ead022 namespace: kserve-ci-e2e-test resourceVersion: "20137" uid: 942702a5-7063-40ac-987f-ac7ae775b11c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:12:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-83116-predictor-849469f6df-2lzrf namespace: kserve-ci-e2e-test resourceVersion: "15702" uid: c5909edf-61bd-432d-bd91-b5e6418f4e8d kind: Event lastTimestamp: "2026-05-20T14:12:33Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T14:12:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:12:33Z" name: error-404-isvc-83116-predictor-849469f6df-2lzrf.18b14b830196740e namespace: kserve-ci-e2e-test resourceVersion: "20054" uid: 59b70dd7-4a05-490a-93b9-58a776bca7a9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:12:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-83116-predictor-849469f6df-2lzrf namespace: kserve-ci-e2e-test resourceVersion: "15702" uid: c5909edf-61bd-432d-bd91-b5e6418f4e8d kind: Event lastTimestamp: "2026-05-20T14:12:33Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:12:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:12:33Z" name: error-404-isvc-83116-predictor-849469f6df-2lzrf.18b14b830197a7f1 namespace: kserve-ci-e2e-test resourceVersion: "20058" uid: bf205108-901f-4952-8fbe-f6dd6dd1f7fa reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:12:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-83116-predictor-849469f6df-2lzrf namespace: kserve-ci-e2e-test resourceVersion: "15702" uid: c5909edf-61bd-432d-bd91-b5e6418f4e8d kind: Event lastTimestamp: "2026-05-20T14:12:36Z" message: 'Readiness probe failed: Get "https://10.134.0.33:8643/healthz": dial tcp 10.134.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-20T14:12:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:12:36Z" name: error-404-isvc-83116-predictor-849469f6df-2lzrf.18b14b83bd6b477e namespace: kserve-ci-e2e-test resourceVersion: "20136" uid: 762cb60f-49d7-43ae-a44e-078b155465bf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:18Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-83116-predictor-849469f6df namespace: kserve-ci-e2e-test resourceVersion: "15691" uid: e58edc7f-f04a-45ba-8d59-365ba887e9dc kind: Event lastTimestamp: "2026-05-20T14:03:18Z" message: 'Created pod: error-404-isvc-83116-predictor-849469f6df-2lzrf' metadata: creationTimestamp: "2026-05-20T14:03:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:03:18Z" name: error-404-isvc-83116-predictor-849469f6df.18b14b01c00ba8ff namespace: kserve-ci-e2e-test resourceVersion: "15703" uid: e73b5df0-4f20-47f6-9775-d987c3e03a0e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:18Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-83116-predictor namespace: kserve-ci-e2e-test resourceVersion: "15688" uid: 28d69bc3-eb74-468e-9050-0a2bb8b449aa kind: Event lastTimestamp: "2026-05-20T14:03:18Z" message: Scaled up replica set error-404-isvc-83116-predictor-849469f6df from 0 to 1 metadata: creationTimestamp: "2026-05-20T14:03:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:03:18Z" name: error-404-isvc-83116-predictor.18b14b01bdf318ae namespace: kserve-ci-e2e-test resourceVersion: "15695" uid: f1363333-30ab-41b6-9bdd-d43acb3edcec reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-20T14:03:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-83116-predictor namespace: kserve-ci-e2e-test resourceVersion: "15690" uid: 4254952c-4004-497a-9da1-ef788ef57b7e kind: Event lastTimestamp: "2026-05-20T14:04:33Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:03:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:04:33Z" name: error-404-isvc-83116-predictor.18b14b053c9823c7 namespace: kserve-ci-e2e-test resourceVersion: "16644" uid: 09af3bfe-eaa1-42c9-b27e-61f157e9c749 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-20T14:03:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-83116-predictor namespace: kserve-ci-e2e-test resourceVersion: "15690" uid: 4254952c-4004-497a-9da1-ef788ef57b7e kind: Event lastTimestamp: "2026-05-20T14:04:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:03:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:04:33Z" name: error-404-isvc-83116-predictor.18b14b053d192724 namespace: kserve-ci-e2e-test resourceVersion: "16645" uid: 5b84e739-4bde-45e1-b0f4-036f8bcdb19d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-83116-predictor namespace: kserve-ci-e2e-test resourceVersion: "15878" uid: 4254952c-4004-497a-9da1-ef788ef57b7e kind: Event lastTimestamp: "2026-05-20T14:03:48Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:03:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:03:48Z" name: error-404-isvc-83116-predictor.18b14b08bc16eac4 namespace: kserve-ci-e2e-test resourceVersion: "16152" uid: 2991cb63-0281-4077-9157-21b96ecdb30f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-83116-predictor namespace: kserve-ci-e2e-test resourceVersion: "15878" uid: 4254952c-4004-497a-9da1-ef788ef57b7e kind: Event lastTimestamp: "2026-05-20T14:03:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:03:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:03:48Z" name: error-404-isvc-83116-predictor.18b14b08bd16371b namespace: kserve-ci-e2e-test resourceVersion: "16160" uid: 0fbe3ccc-5b57-463b-8b81-241835c38c2e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-20T14:03:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-83116 namespace: kserve-ci-e2e-test resourceVersion: "15659" uid: 746bc480-eaaa-4367-86cb-2f2ba7a8b8bd kind: Event lastTimestamp: "2026-05-20T14:04:07Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T14:03:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:04:07Z" name: error-404-isvc-83116.18b14b01bdba6a7a namespace: kserve-ci-e2e-test resourceVersion: "16379" uid: 558a117d-8420-44eb-af43-8b8acda14453 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-83116 namespace: kserve-ci-e2e-test resourceVersion: "15659" uid: 746bc480-eaaa-4367-86cb-2f2ba7a8b8bd kind: Event lastTimestamp: "2026-05-20T14:03:18Z" message: 'Failed to update status for InferenceService "error-404-isvc-83116": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-83116": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T14:03:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:03:18Z" name: error-404-isvc-83116.18b14b01be38ef22 namespace: kserve-ci-e2e-test resourceVersion: "15697" uid: a924e502-ffed-4850-a50b-db00b383d6e7 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-83116 namespace: kserve-ci-e2e-test resourceVersion: "15659" uid: 746bc480-eaaa-4367-86cb-2f2ba7a8b8bd kind: Event lastTimestamp: "2026-05-20T14:03:18Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-83116": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T14:03:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:03:18Z" name: error-404-isvc-83116.18b14b01be394503 namespace: kserve-ci-e2e-test resourceVersion: "15698" uid: b7166dbf-cf74-470f-ba68-45439fefb2aa reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:04:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-83116 namespace: kserve-ci-e2e-test resourceVersion: "16375" uid: 746bc480-eaaa-4367-86cb-2f2ba7a8b8bd kind: Event lastTimestamp: "2026-05-20T14:04:07Z" message: InferenceService [error-404-isvc-83116] is Ready metadata: creationTimestamp: "2026-05-20T14:04:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:04:07Z" name: error-404-isvc-83116.18b14b0d06c73ced namespace: kserve-ci-e2e-test resourceVersion: "16378" uid: d57d86e4-6235-4b9f-b666-5225b2a0559a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T14:22:58.903362Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a868a-predictor-848b5648c5-46cjq namespace: kserve-ci-e2e-test resourceVersion: "25317" uid: b151f8aa-7bf0-4a45-8612-a746025bcae7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-a868a-predictor-848b5648c5-46cjq to ip-10-0-136-75.ec2.internal metadata: creationTimestamp: "2026-05-20T14:22:58Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T14:22:58Z" name: error-404-isvc-a868a-predictor-848b5648c5-46cjq.18b14c148b037282 namespace: kserve-ci-e2e-test resourceVersion: "25321" uid: e2d7950b-b9ee-468c-b484-56b8d5533775 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-747dc7bb84-jx2kx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:22:59Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a868a-predictor-848b5648c5-46cjq namespace: kserve-ci-e2e-test resourceVersion: "25318" uid: b151f8aa-7bf0-4a45-8612-a746025bcae7 kind: Event lastTimestamp: "2026-05-20T14:22:59Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-a868a-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-20T14:22:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:22:59Z" name: error-404-isvc-a868a-predictor-848b5648c5-46cjq.18b14c14991e63be namespace: kserve-ci-e2e-test resourceVersion: "25332" uid: 49b8adf8-a6cd-4c73-b50a-db98391c496a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:22:59Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a868a-predictor-848b5648c5-46cjq namespace: kserve-ci-e2e-test resourceVersion: "25326" uid: b151f8aa-7bf0-4a45-8612-a746025bcae7 kind: Event lastTimestamp: "2026-05-20T14:22:59Z" message: Add eth0 [10.134.0.46/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T14:22:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T14:22:59Z" name: error-404-isvc-a868a-predictor-848b5648c5-46cjq.18b14c14c8ab47dc namespace: kserve-ci-e2e-test resourceVersion: "25349" uid: 6d37a730-b1d6-4fc0-8ca8-7bca1065bda3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:22:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a868a-predictor-848b5648c5-46cjq namespace: kserve-ci-e2e-test resourceVersion: "25318" uid: b151f8aa-7bf0-4a45-8612-a746025bcae7 kind: Event lastTimestamp: "2026-05-20T14:22:59Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1501" already present on machine metadata: creationTimestamp: "2026-05-20T14:22:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:22:59Z" name: error-404-isvc-a868a-predictor-848b5648c5-46cjq.18b14c14ca7c3ecd namespace: kserve-ci-e2e-test resourceVersion: "25353" uid: a5af09b3-ff89-4b19-b0fe-79ca67b0000c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:23:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a868a-predictor-848b5648c5-46cjq namespace: kserve-ci-e2e-test resourceVersion: "25318" uid: b151f8aa-7bf0-4a45-8612-a746025bcae7 kind: Event lastTimestamp: "2026-05-20T14:23:00Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T14:23:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:23:00Z" name: error-404-isvc-a868a-predictor-848b5648c5-46cjq.18b14c14cdbb9b81 namespace: kserve-ci-e2e-test resourceVersion: "25355" uid: ae14d75b-b8df-494c-8e22-49729925b009 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:23:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a868a-predictor-848b5648c5-46cjq namespace: kserve-ci-e2e-test resourceVersion: "25318" uid: b151f8aa-7bf0-4a45-8612-a746025bcae7 kind: Event lastTimestamp: "2026-05-20T14:23:00Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T14:23:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:23:00Z" name: error-404-isvc-a868a-predictor-848b5648c5-46cjq.18b14c14ce3d3914 namespace: kserve-ci-e2e-test resourceVersion: "25356" uid: 2c3cd751-b2f9-443c-aaf5-52b7d09dbf99 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:23:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a868a-predictor-848b5648c5-46cjq namespace: kserve-ci-e2e-test resourceVersion: "25318" uid: b151f8aa-7bf0-4a45-8612-a746025bcae7 kind: Event lastTimestamp: "2026-05-20T14:23:00Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-20T14:23:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:23:00Z" name: error-404-isvc-a868a-predictor-848b5648c5-46cjq.18b14c14ce4b62f9 namespace: kserve-ci-e2e-test resourceVersion: "25357" uid: fda2f81f-e01d-4ede-984f-4cf2a7f954e0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:23:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a868a-predictor-848b5648c5-46cjq namespace: kserve-ci-e2e-test resourceVersion: "25318" uid: b151f8aa-7bf0-4a45-8612-a746025bcae7 kind: Event lastTimestamp: "2026-05-20T14:23:00Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T14:23:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:23:00Z" name: error-404-isvc-a868a-predictor-848b5648c5-46cjq.18b14c14d1952d36 namespace: kserve-ci-e2e-test resourceVersion: "25358" uid: 028ac99c-49a5-4c95-a117-624441de609f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:23:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a868a-predictor-848b5648c5-46cjq namespace: kserve-ci-e2e-test resourceVersion: "25318" uid: b151f8aa-7bf0-4a45-8612-a746025bcae7 kind: Event lastTimestamp: "2026-05-20T14:23:00Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:23:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:23:00Z" name: error-404-isvc-a868a-predictor-848b5648c5-46cjq.18b14c14d20ed696 namespace: kserve-ci-e2e-test resourceVersion: "25359" uid: 429a9a85-b1dc-4365-b967-2ccad52a0c66 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-20T14:23:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a868a-predictor-848b5648c5-46cjq namespace: kserve-ci-e2e-test resourceVersion: "25318" uid: b151f8aa-7bf0-4a45-8612-a746025bcae7 kind: Event lastTimestamp: "2026-05-20T14:23:37Z" message: 'Readiness probe failed: dial tcp 10.134.0.46:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T14:23:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:23:37Z" name: error-404-isvc-a868a-predictor-848b5648c5-46cjq.18b14c150ed41cf4 namespace: kserve-ci-e2e-test resourceVersion: "25807" uid: 8d8f5e30-50f7-4558-8a08-0017bd608444 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:24:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a868a-predictor-848b5648c5-46cjq namespace: kserve-ci-e2e-test resourceVersion: "25318" uid: b151f8aa-7bf0-4a45-8612-a746025bcae7 kind: Event lastTimestamp: "2026-05-20T14:24:09Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T14:24:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:24:09Z" name: error-404-isvc-a868a-predictor-848b5648c5-46cjq.18b14c24e92c4c08 namespace: kserve-ci-e2e-test resourceVersion: "26507" uid: b8c988cf-e03d-401b-964c-431da466514e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:24:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a868a-predictor-848b5648c5-46cjq namespace: kserve-ci-e2e-test resourceVersion: "25318" uid: b151f8aa-7bf0-4a45-8612-a746025bcae7 kind: Event lastTimestamp: "2026-05-20T14:24:09Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:24:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:24:09Z" name: error-404-isvc-a868a-predictor-848b5648c5-46cjq.18b14c24e92fcf36 namespace: kserve-ci-e2e-test resourceVersion: "26509" uid: 2d140c94-957e-4e66-a0e8-b28ecbd88a25 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:24:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a868a-predictor-848b5648c5-46cjq namespace: kserve-ci-e2e-test resourceVersion: "25318" uid: b151f8aa-7bf0-4a45-8612-a746025bcae7 kind: Event lastTimestamp: "2026-05-20T14:24:12Z" message: 'Readiness probe failed: Get "https://10.134.0.46:8643/healthz": dial tcp 10.134.0.46:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-20T14:24:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:24:12Z" name: error-404-isvc-a868a-predictor-848b5648c5-46cjq.18b14c2596f14473 namespace: kserve-ci-e2e-test resourceVersion: "26578" uid: d27992ab-0186-4e02-a647-87c1e7b3e57c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:22:58Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-a868a-predictor-848b5648c5 namespace: kserve-ci-e2e-test resourceVersion: "25309" uid: 73ca1041-33f0-4fed-92bc-ad799d0faccd kind: Event lastTimestamp: "2026-05-20T14:22:58Z" message: 'Created pod: error-404-isvc-a868a-predictor-848b5648c5-46cjq' metadata: creationTimestamp: "2026-05-20T14:22:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:22:58Z" name: error-404-isvc-a868a-predictor-848b5648c5.18b14c148aa68d03 namespace: kserve-ci-e2e-test resourceVersion: "25319" uid: f9061612-0b07-4c1e-9ae1-9924bba3d12f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:22:58Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-a868a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25305" uid: 83cbc908-cb57-42c2-abdd-8839e8310683 kind: Event lastTimestamp: "2026-05-20T14:22:58Z" message: Scaled up replica set error-404-isvc-a868a-predictor-848b5648c5 from 0 to 1 metadata: creationTimestamp: "2026-05-20T14:22:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:22:58Z" name: error-404-isvc-a868a-predictor.18b14c1488de1e72 namespace: kserve-ci-e2e-test resourceVersion: "25312" uid: 0ed61966-e6bc-4b20-82bc-7587d8f81d97 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T14:23:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a868a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25308" uid: 167eb35d-2296-46f1-9111-762a7fb7b52f kind: Event lastTimestamp: "2026-05-20T14:23:43Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:23:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:23:43Z" name: error-404-isvc-a868a-predictor.18b14c18076bbcdf namespace: kserve-ci-e2e-test resourceVersion: "26079" uid: 807858da-e301-4cb7-9ec3-01e58cf0b6ff reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T14:23:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a868a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25308" uid: 167eb35d-2296-46f1-9111-762a7fb7b52f kind: Event lastTimestamp: "2026-05-20T14:23:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:23:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:23:43Z" name: error-404-isvc-a868a-predictor.18b14c1807df00c7 namespace: kserve-ci-e2e-test resourceVersion: "26080" uid: db2fadc4-91cc-4050-bf3e-e4db0792cdbf reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:23:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a868a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25481" uid: 167eb35d-2296-46f1-9111-762a7fb7b52f kind: Event lastTimestamp: "2026-05-20T14:23:58Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:23:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:23:58Z" name: error-404-isvc-a868a-predictor.18b14c228487215c namespace: kserve-ci-e2e-test resourceVersion: "26318" uid: 3b186c6f-3282-4138-aeb4-5ebfcbd8ad62 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:23:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a868a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25481" uid: 167eb35d-2296-46f1-9111-762a7fb7b52f kind: Event lastTimestamp: "2026-05-20T14:23:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:23:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:23:58Z" name: error-404-isvc-a868a-predictor.18b14c2284fb7383 namespace: kserve-ci-e2e-test resourceVersion: "26321" uid: 0c1824d8-c426-4661-98f8-f5cec68c837e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-20T14:22:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a868a namespace: kserve-ci-e2e-test resourceVersion: "25276" uid: e392c26b-ee5f-4373-a411-0a4e0366c8c9 kind: Event lastTimestamp: "2026-05-20T14:23:47Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T14:22:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:23:47Z" name: error-404-isvc-a868a.18b14c1488abde5f namespace: kserve-ci-e2e-test resourceVersion: "26200" uid: 5e6888d6-083b-4ad2-bcd7-9512af544f98 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:22:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a868a namespace: kserve-ci-e2e-test resourceVersion: "25276" uid: e392c26b-ee5f-4373-a411-0a4e0366c8c9 kind: Event lastTimestamp: "2026-05-20T14:22:58Z" message: 'Failed to update status for InferenceService "error-404-isvc-a868a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-a868a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T14:22:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:22:58Z" name: error-404-isvc-a868a.18b14c1489022fa2 namespace: kserve-ci-e2e-test resourceVersion: "25314" uid: c883dee9-04bb-4141-859a-6e8f8ee3949f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:22:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a868a namespace: kserve-ci-e2e-test resourceVersion: "25276" uid: e392c26b-ee5f-4373-a411-0a4e0366c8c9 kind: Event lastTimestamp: "2026-05-20T14:22:58Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-a868a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T14:22:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:22:58Z" name: error-404-isvc-a868a.18b14c14890281d8 namespace: kserve-ci-e2e-test resourceVersion: "25315" uid: 63ca4af4-d289-4641-9efa-c0a00e31256c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:23:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a868a namespace: kserve-ci-e2e-test resourceVersion: "26195" uid: e392c26b-ee5f-4373-a411-0a4e0366c8c9 kind: Event lastTimestamp: "2026-05-20T14:23:47Z" message: InferenceService [error-404-isvc-a868a] is Ready metadata: creationTimestamp: "2026-05-20T14:23:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:23:47Z" name: error-404-isvc-a868a.18b14c1fcb13df85 namespace: kserve-ci-e2e-test resourceVersion: "26197" uid: 03ec179f-cdc6-40da-888c-9cb4537b4b01 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T14:03:48.687662Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-cdd1d-predictor-66cb4945ff-2r8ml namespace: kserve-ci-e2e-test resourceVersion: "16144" uid: 25bf4864-4618-4763-8476-04e940c2ed2c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-cdd1d-predictor-66cb4945ff-2r8ml to ip-10-0-136-75.ec2.internal metadata: creationTimestamp: "2026-05-20T14:03:48Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T14:03:48Z" name: error-404-isvc-cdd1d-predictor-66cb4945ff-2r8ml.18b14b08bcd0ae3f namespace: kserve-ci-e2e-test resourceVersion: "16158" uid: c1de7902-ff34-4cba-bfa5-9871e4864642 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-747dc7bb84-jx2kx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:48Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-cdd1d-predictor-66cb4945ff-2r8ml namespace: kserve-ci-e2e-test resourceVersion: "16154" uid: 25bf4864-4618-4763-8476-04e940c2ed2c kind: Event lastTimestamp: "2026-05-20T14:03:48Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-cdd1d-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-20T14:03:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:48Z" name: error-404-isvc-cdd1d-predictor-66cb4945ff-2r8ml.18b14b08ca069f5b namespace: kserve-ci-e2e-test resourceVersion: "16176" uid: fc742d1c-4b87-4460-8a99-d60bad8cc160 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:49Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-cdd1d-predictor-66cb4945ff-2r8ml namespace: kserve-ci-e2e-test resourceVersion: "16169" uid: 25bf4864-4618-4763-8476-04e940c2ed2c kind: Event lastTimestamp: "2026-05-20T14:03:49Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T14:03:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T14:03:49Z" name: error-404-isvc-cdd1d-predictor-66cb4945ff-2r8ml.18b14b08fb00221c namespace: kserve-ci-e2e-test resourceVersion: "16200" uid: 635c91f2-89a7-49dc-8ac4-bbf3dbcf8d79 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cdd1d-predictor-66cb4945ff-2r8ml namespace: kserve-ci-e2e-test resourceVersion: "16154" uid: 25bf4864-4618-4763-8476-04e940c2ed2c kind: Event lastTimestamp: "2026-05-20T14:03:49Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1501" already present on machine metadata: creationTimestamp: "2026-05-20T14:03:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:49Z" name: error-404-isvc-cdd1d-predictor-66cb4945ff-2r8ml.18b14b08fc8352a6 namespace: kserve-ci-e2e-test resourceVersion: "16204" uid: 20ba8a8c-97cd-413c-94d4-b4ec7b38a54e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cdd1d-predictor-66cb4945ff-2r8ml namespace: kserve-ci-e2e-test resourceVersion: "16154" uid: 25bf4864-4618-4763-8476-04e940c2ed2c kind: Event lastTimestamp: "2026-05-20T14:03:49Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T14:03:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:49Z" name: error-404-isvc-cdd1d-predictor-66cb4945ff-2r8ml.18b14b08ff9f4210 namespace: kserve-ci-e2e-test resourceVersion: "16206" uid: 51b88e1b-2fd5-45a9-b452-23ff71bae2eb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cdd1d-predictor-66cb4945ff-2r8ml namespace: kserve-ci-e2e-test resourceVersion: "16154" uid: 25bf4864-4618-4763-8476-04e940c2ed2c kind: Event lastTimestamp: "2026-05-20T14:03:49Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T14:03:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:49Z" name: error-404-isvc-cdd1d-predictor-66cb4945ff-2r8ml.18b14b09000a4839 namespace: kserve-ci-e2e-test resourceVersion: "16207" uid: b11ea35b-f487-4520-b78d-49a93e060513 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-cdd1d-predictor-66cb4945ff-2r8ml namespace: kserve-ci-e2e-test resourceVersion: "16154" uid: 25bf4864-4618-4763-8476-04e940c2ed2c kind: Event lastTimestamp: "2026-05-20T14:03:49Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-20T14:03:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:49Z" name: error-404-isvc-cdd1d-predictor-66cb4945ff-2r8ml.18b14b0900174d31 namespace: kserve-ci-e2e-test resourceVersion: "16208" uid: c117c4fe-e04e-41f7-ab48-de7c53df155d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-cdd1d-predictor-66cb4945ff-2r8ml namespace: kserve-ci-e2e-test resourceVersion: "16154" uid: 25bf4864-4618-4763-8476-04e940c2ed2c kind: Event lastTimestamp: "2026-05-20T14:03:49Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T14:03:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:49Z" name: error-404-isvc-cdd1d-predictor-66cb4945ff-2r8ml.18b14b0903c4d323 namespace: kserve-ci-e2e-test resourceVersion: "16210" uid: f722a6dc-f882-48f1-b8e2-52d8bcb53b65 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-cdd1d-predictor-66cb4945ff-2r8ml namespace: kserve-ci-e2e-test resourceVersion: "16154" uid: 25bf4864-4618-4763-8476-04e940c2ed2c kind: Event lastTimestamp: "2026-05-20T14:03:49Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:03:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:49Z" name: error-404-isvc-cdd1d-predictor-66cb4945ff-2r8ml.18b14b0904749f11 namespace: kserve-ci-e2e-test resourceVersion: "16211" uid: 31778f01-3fcd-4d03-bc28-2897ac48a58b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-20T14:03:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cdd1d-predictor-66cb4945ff-2r8ml namespace: kserve-ci-e2e-test resourceVersion: "16154" uid: 25bf4864-4618-4763-8476-04e940c2ed2c kind: Event lastTimestamp: "2026-05-20T14:04:27Z" message: 'Readiness probe failed: dial tcp 10.134.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T14:03:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:04:27Z" name: error-404-isvc-cdd1d-predictor-66cb4945ff-2r8ml.18b14b094bcd5ea3 namespace: kserve-ci-e2e-test resourceVersion: "16586" uid: 36682961-f0fa-45bc-9222-0aaecf5d57e4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cdd1d-predictor-66cb4945ff-2r8ml namespace: kserve-ci-e2e-test resourceVersion: "16154" uid: 25bf4864-4618-4763-8476-04e940c2ed2c kind: Event lastTimestamp: "2026-05-20T14:13:03Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T14:13:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:13:03Z" name: error-404-isvc-cdd1d-predictor-66cb4945ff-2r8ml.18b14b89f152a2b6 namespace: kserve-ci-e2e-test resourceVersion: "20424" uid: 9f143cee-d7d7-47cb-9bb3-5897bdfd9519 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-cdd1d-predictor-66cb4945ff-2r8ml namespace: kserve-ci-e2e-test resourceVersion: "16154" uid: 25bf4864-4618-4763-8476-04e940c2ed2c kind: Event lastTimestamp: "2026-05-20T14:13:03Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:13:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:13:03Z" name: error-404-isvc-cdd1d-predictor-66cb4945ff-2r8ml.18b14b89f1533254 namespace: kserve-ci-e2e-test resourceVersion: "20427" uid: d619fad3-5309-45c8-b34c-2a2c6c2a3cf8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:48Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-cdd1d-predictor-66cb4945ff namespace: kserve-ci-e2e-test resourceVersion: "16135" uid: fbcfa2f7-585c-4859-afad-5a19ba185b1d kind: Event lastTimestamp: "2026-05-20T14:03:48Z" message: 'Created pod: error-404-isvc-cdd1d-predictor-66cb4945ff-2r8ml' metadata: creationTimestamp: "2026-05-20T14:03:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:03:48Z" name: error-404-isvc-cdd1d-predictor-66cb4945ff.18b14b08bc4ef824 namespace: kserve-ci-e2e-test resourceVersion: "16155" uid: c930ac5b-81d5-4a02-85c8-259982585c58 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:48Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-cdd1d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16131" uid: 1153b847-cb4a-4100-ba0d-510aa6377d45 kind: Event lastTimestamp: "2026-05-20T14:03:48Z" message: Scaled up replica set error-404-isvc-cdd1d-predictor-66cb4945ff from 0 to 1 metadata: creationTimestamp: "2026-05-20T14:03:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:03:48Z" name: error-404-isvc-cdd1d-predictor.18b14b08ba68e329 namespace: kserve-ci-e2e-test resourceVersion: "16136" uid: fbece225-2884-4d30-93f9-4d6ff388ba19 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T14:04:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-cdd1d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16134" uid: a78e1256-24ff-4309-a636-7d12f3757756 kind: Event lastTimestamp: "2026-05-20T14:04:18Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:04:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:04:18Z" name: error-404-isvc-cdd1d-predictor.18b14b0c39aba7e1 namespace: kserve-ci-e2e-test resourceVersion: "16513" uid: 34f27e80-7681-4085-bb21-619209ddddc8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T14:04:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-cdd1d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16134" uid: a78e1256-24ff-4309-a636-7d12f3757756 kind: Event lastTimestamp: "2026-05-20T14:04:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:04:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:04:18Z" name: error-404-isvc-cdd1d-predictor.18b14b0c3a585b02 namespace: kserve-ci-e2e-test resourceVersion: "16514" uid: d37cfca7-cac4-461c-88eb-a9aa9e424048 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T14:04:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-cdd1d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16338" uid: a78e1256-24ff-4309-a636-7d12f3757756 kind: Event lastTimestamp: "2026-05-20T14:05:03Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:04:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:05:03Z" name: error-404-isvc-cdd1d-predictor.18b14b1338524cbe namespace: kserve-ci-e2e-test resourceVersion: "16923" uid: cbeeee0b-dcf1-46b5-b3ea-da07d18aeb82 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T14:04:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-cdd1d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16338" uid: a78e1256-24ff-4309-a636-7d12f3757756 kind: Event lastTimestamp: "2026-05-20T14:05:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:04:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:05:03Z" name: error-404-isvc-cdd1d-predictor.18b14b1338eef18f namespace: kserve-ci-e2e-test resourceVersion: "16924" uid: 29367de5-619f-4900-9055-0d5e593c1c81 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-20T14:03:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-cdd1d namespace: kserve-ci-e2e-test resourceVersion: "16101" uid: 11685393-7efe-48c1-ba4d-608936742abe kind: Event lastTimestamp: "2026-05-20T14:04:37Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T14:03:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:04:37Z" name: error-404-isvc-cdd1d.18b14b08ba700a6e namespace: kserve-ci-e2e-test resourceVersion: "16680" uid: 26cbd28f-a14d-445e-a072-baf70c669d56 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-cdd1d namespace: kserve-ci-e2e-test resourceVersion: "16101" uid: 11685393-7efe-48c1-ba4d-608936742abe kind: Event lastTimestamp: "2026-05-20T14:03:48Z" message: 'Failed to update status for InferenceService "error-404-isvc-cdd1d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-cdd1d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T14:03:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:03:48Z" name: error-404-isvc-cdd1d.18b14b08bb098060 namespace: kserve-ci-e2e-test resourceVersion: "16142" uid: 9f840e3f-e675-4ab8-913c-396fb8530506 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-cdd1d namespace: kserve-ci-e2e-test resourceVersion: "16101" uid: 11685393-7efe-48c1-ba4d-608936742abe kind: Event lastTimestamp: "2026-05-20T14:03:48Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-cdd1d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T14:03:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:03:48Z" name: error-404-isvc-cdd1d.18b14b08bb09d881 namespace: kserve-ci-e2e-test resourceVersion: "16143" uid: 4082c930-dcf9-4bc5-a161-96485b0df121 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:04:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-cdd1d namespace: kserve-ci-e2e-test resourceVersion: "16677" uid: 11685393-7efe-48c1-ba4d-608936742abe kind: Event lastTimestamp: "2026-05-20T14:04:37Z" message: InferenceService [error-404-isvc-cdd1d] is Ready metadata: creationTimestamp: "2026-05-20T14:04:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:04:37Z" name: error-404-isvc-cdd1d.18b14b140963aa4d namespace: kserve-ci-e2e-test resourceVersion: "16678" uid: aa772fb0-58f5-4e7a-b334-5c9fca693bf1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T14:13:44.070772Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ecafe-predictor-bf7d7ddc-5k2k4 namespace: kserve-ci-e2e-test resourceVersion: "20957" uid: c82902d1-af79-484e-a67e-1f0ed4b08fc9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-ecafe-predictor-bf7d7ddc-5k2k4 to ip-10-0-136-75.ec2.internal metadata: creationTimestamp: "2026-05-20T14:13:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T14:13:44Z" name: error-404-isvc-ecafe-predictor-bf7d7ddc-5k2k4.18b14b935c69ff12 namespace: kserve-ci-e2e-test resourceVersion: "20961" uid: 79a80761-c964-411e-841a-4e78f4fea210 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-747dc7bb84-jx2kx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:44Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ecafe-predictor-bf7d7ddc-5k2k4 namespace: kserve-ci-e2e-test resourceVersion: "20965" uid: c82902d1-af79-484e-a67e-1f0ed4b08fc9 kind: Event lastTimestamp: "2026-05-20T14:13:44Z" message: Add eth0 [10.134.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T14:13:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T14:13:44Z" name: error-404-isvc-ecafe-predictor-bf7d7ddc-5k2k4.18b14b93768b0055 namespace: kserve-ci-e2e-test resourceVersion: "20983" uid: 4e3e054f-9155-486e-90fa-1099f718e425 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ecafe-predictor-bf7d7ddc-5k2k4 namespace: kserve-ci-e2e-test resourceVersion: "20959" uid: c82902d1-af79-484e-a67e-1f0ed4b08fc9 kind: Event lastTimestamp: "2026-05-20T14:13:44Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1501" already present on machine metadata: creationTimestamp: "2026-05-20T14:13:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:13:44Z" name: error-404-isvc-ecafe-predictor-bf7d7ddc-5k2k4.18b14b9377ee2cb5 namespace: kserve-ci-e2e-test resourceVersion: "20985" uid: 43c203cd-904b-46bf-a1bf-9e7ecaf73e99 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ecafe-predictor-bf7d7ddc-5k2k4 namespace: kserve-ci-e2e-test resourceVersion: "20959" uid: c82902d1-af79-484e-a67e-1f0ed4b08fc9 kind: Event lastTimestamp: "2026-05-20T14:13:44Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T14:13:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:13:44Z" name: error-404-isvc-ecafe-predictor-bf7d7ddc-5k2k4.18b14b937b161454 namespace: kserve-ci-e2e-test resourceVersion: "20986" uid: 14e77601-4c49-45d7-aae6-b8ebd73d0698 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ecafe-predictor-bf7d7ddc-5k2k4 namespace: kserve-ci-e2e-test resourceVersion: "20959" uid: c82902d1-af79-484e-a67e-1f0ed4b08fc9 kind: Event lastTimestamp: "2026-05-20T14:13:44Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T14:13:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:13:44Z" name: error-404-isvc-ecafe-predictor-bf7d7ddc-5k2k4.18b14b937b7e4a8a namespace: kserve-ci-e2e-test resourceVersion: "20987" uid: e9c72b63-1fe5-4ff3-8654-3dda2a3a071f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ecafe-predictor-bf7d7ddc-5k2k4 namespace: kserve-ci-e2e-test resourceVersion: "20959" uid: c82902d1-af79-484e-a67e-1f0ed4b08fc9 kind: Event lastTimestamp: "2026-05-20T14:13:44Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-20T14:13:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:13:44Z" name: error-404-isvc-ecafe-predictor-bf7d7ddc-5k2k4.18b14b937b8b4795 namespace: kserve-ci-e2e-test resourceVersion: "20988" uid: 36883c62-6a1d-4c8b-8104-77df09865b26 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ecafe-predictor-bf7d7ddc-5k2k4 namespace: kserve-ci-e2e-test resourceVersion: "20959" uid: c82902d1-af79-484e-a67e-1f0ed4b08fc9 kind: Event lastTimestamp: "2026-05-20T14:13:44Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T14:13:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:13:44Z" name: error-404-isvc-ecafe-predictor-bf7d7ddc-5k2k4.18b14b937e0545fc namespace: kserve-ci-e2e-test resourceVersion: "20992" uid: 95227c17-c59b-480d-ad86-59cb0e5c213d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ecafe-predictor-bf7d7ddc-5k2k4 namespace: kserve-ci-e2e-test resourceVersion: "20959" uid: c82902d1-af79-484e-a67e-1f0ed4b08fc9 kind: Event lastTimestamp: "2026-05-20T14:13:44Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:13:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:13:44Z" name: error-404-isvc-ecafe-predictor-bf7d7ddc-5k2k4.18b14b937ebb0e4c namespace: kserve-ci-e2e-test resourceVersion: "20994" uid: 2196e402-8423-4b2f-891a-a5be209a0603 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-05-20T14:13:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ecafe-predictor-bf7d7ddc-5k2k4 namespace: kserve-ci-e2e-test resourceVersion: "20959" uid: c82902d1-af79-484e-a67e-1f0ed4b08fc9 kind: Event lastTimestamp: "2026-05-20T14:23:02Z" message: 'Readiness probe failed: dial tcp 10.134.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T14:13:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:23:02Z" name: error-404-isvc-ecafe-predictor-bf7d7ddc-5k2k4.18b14b93da810e82 namespace: kserve-ci-e2e-test resourceVersion: "25388" uid: dea70619-db82-42de-a71e-eacd14be5a7b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:22:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ecafe-predictor-bf7d7ddc-5k2k4 namespace: kserve-ci-e2e-test resourceVersion: "20959" uid: c82902d1-af79-484e-a67e-1f0ed4b08fc9 kind: Event lastTimestamp: "2026-05-20T14:22:58Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T14:22:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:22:58Z" name: error-404-isvc-ecafe-predictor-bf7d7ddc-5k2k4.18b14c1487ab2641 namespace: kserve-ci-e2e-test resourceVersion: "25307" uid: 49af18ff-1a62-4f59-9c8c-779b671dcc16 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:22:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ecafe-predictor-bf7d7ddc-5k2k4 namespace: kserve-ci-e2e-test resourceVersion: "20959" uid: c82902d1-af79-484e-a67e-1f0ed4b08fc9 kind: Event lastTimestamp: "2026-05-20T14:22:58Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:22:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:22:58Z" name: error-404-isvc-ecafe-predictor-bf7d7ddc-5k2k4.18b14c1487abaa8b namespace: kserve-ci-e2e-test resourceVersion: "25310" uid: a3677c7a-0c48-488e-936e-2effe8dbe8ef reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:23:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ecafe-predictor-bf7d7ddc-5k2k4 namespace: kserve-ci-e2e-test resourceVersion: "20959" uid: c82902d1-af79-484e-a67e-1f0ed4b08fc9 kind: Event lastTimestamp: "2026-05-20T14:23:02Z" message: 'Readiness probe failed: Get "https://10.134.0.41:8643/healthz": dial tcp 10.134.0.41:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-20T14:23:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:23:02Z" name: error-404-isvc-ecafe-predictor-bf7d7ddc-5k2k4.18b14c154ee6ab0d namespace: kserve-ci-e2e-test resourceVersion: "25387" uid: 46bf346e-3c7e-45db-8485-3e70239ec219 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-ecafe-predictor-bf7d7ddc namespace: kserve-ci-e2e-test resourceVersion: "20949" uid: 3f7e42a4-de19-4f53-9b71-457634f4a210 kind: Event lastTimestamp: "2026-05-20T14:13:44Z" message: 'Created pod: error-404-isvc-ecafe-predictor-bf7d7ddc-5k2k4' metadata: creationTimestamp: "2026-05-20T14:13:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:13:44Z" name: error-404-isvc-ecafe-predictor-bf7d7ddc.18b14b935bd6e332 namespace: kserve-ci-e2e-test resourceVersion: "20958" uid: 060951c4-7a05-4ec3-b611-4f12ede46669 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-ecafe-predictor namespace: kserve-ci-e2e-test resourceVersion: "20945" uid: 9802b3d1-73fb-4762-b523-4220fc051704 kind: Event lastTimestamp: "2026-05-20T14:13:44Z" message: Scaled up replica set error-404-isvc-ecafe-predictor-bf7d7ddc from 0 to 1 metadata: creationTimestamp: "2026-05-20T14:13:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:13:44Z" name: error-404-isvc-ecafe-predictor.18b14b9359914e1b namespace: kserve-ci-e2e-test resourceVersion: "20952" uid: 1a88487a-a37f-4c3a-ac21-20f7b5ed4e08 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T14:13:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ecafe-predictor namespace: kserve-ci-e2e-test resourceVersion: "20948" uid: 90c4b744-b765-40bb-aa78-2520603f8474 kind: Event lastTimestamp: "2026-05-20T14:14:29Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:13:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:14:29Z" name: error-404-isvc-ecafe-predictor.18b14b96d8f41d57 namespace: kserve-ci-e2e-test resourceVersion: "21573" uid: c928f092-d7d9-44b6-9b3f-af0ca4574fc0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T14:13:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ecafe-predictor namespace: kserve-ci-e2e-test resourceVersion: "20948" uid: 90c4b744-b765-40bb-aa78-2520603f8474 kind: Event lastTimestamp: "2026-05-20T14:14:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:13:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:14:29Z" name: error-404-isvc-ecafe-predictor.18b14b96d9a3d121 namespace: kserve-ci-e2e-test resourceVersion: "21574" uid: 1252ec59-ddf4-4e01-bd0f-e4e0317cbbe4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T14:14:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ecafe-predictor namespace: kserve-ci-e2e-test resourceVersion: "21140" uid: 90c4b744-b765-40bb-aa78-2520603f8474 kind: Event lastTimestamp: "2026-05-20T14:14:59Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:14:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:14:59Z" name: error-404-isvc-ecafe-predictor.18b14b9a58df8607 namespace: kserve-ci-e2e-test resourceVersion: "21873" uid: c7ada733-ba78-430f-819b-e257fba8420d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T14:14:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ecafe-predictor namespace: kserve-ci-e2e-test resourceVersion: "21140" uid: 90c4b744-b765-40bb-aa78-2520603f8474 kind: Event lastTimestamp: "2026-05-20T14:14:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:14:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:14:59Z" name: error-404-isvc-ecafe-predictor.18b14b9a59c8cd99 namespace: kserve-ci-e2e-test resourceVersion: "21874" uid: b2551b1f-8ed4-4c2a-828d-38cfb5c56728 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-20T14:13:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ecafe namespace: kserve-ci-e2e-test resourceVersion: "20914" uid: 1ddbdfe9-e8e8-42b7-9641-9b58f8b3b28e kind: Event lastTimestamp: "2026-05-20T14:14:32Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T14:13:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:14:32Z" name: error-404-isvc-ecafe.18b14b93594ca505 namespace: kserve-ci-e2e-test resourceVersion: "21619" uid: 3e373320-e0af-48cc-a91a-2c81f090fbd9 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ecafe namespace: kserve-ci-e2e-test resourceVersion: "20914" uid: 1ddbdfe9-e8e8-42b7-9641-9b58f8b3b28e kind: Event lastTimestamp: "2026-05-20T14:13:44Z" message: 'Failed to update status for InferenceService "error-404-isvc-ecafe": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-ecafe": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T14:13:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:13:44Z" name: error-404-isvc-ecafe.18b14b9359fbbd05 namespace: kserve-ci-e2e-test resourceVersion: "20954" uid: b5cb0c06-d770-4819-962e-da1a0f11e418 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ecafe namespace: kserve-ci-e2e-test resourceVersion: "20914" uid: 1ddbdfe9-e8e8-42b7-9641-9b58f8b3b28e kind: Event lastTimestamp: "2026-05-20T14:13:44Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-ecafe": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T14:13:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:13:44Z" name: error-404-isvc-ecafe.18b14b9359fc31d6 namespace: kserve-ci-e2e-test resourceVersion: "20955" uid: e599bbf0-8ee4-472f-a2b6-ac63b5052411 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:14:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ecafe namespace: kserve-ci-e2e-test resourceVersion: "21617" uid: 1ddbdfe9-e8e8-42b7-9641-9b58f8b3b28e kind: Event lastTimestamp: "2026-05-20T14:14:32Z" message: InferenceService [error-404-isvc-ecafe] is Ready metadata: creationTimestamp: "2026-05-20T14:14:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:14:32Z" name: error-404-isvc-ecafe.18b14b9e99ec9a9c namespace: kserve-ci-e2e-test resourceVersion: "21618" uid: 30c259bf-bc92-47a1-b579-567f0b3ca6ab reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T14:01:38.140519Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-664c5c98bd-8nr4d namespace: kserve-ci-e2e-test resourceVersion: "14507" uid: efd1378d-8fc2-43b0-9a18-4cce7fac91f9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-664c5c98bd-8nr4d to ip-10-0-137-148.ec2.internal metadata: creationTimestamp: "2026-05-20T14:01:38Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T14:01:38Z" name: isvc-sklearn-graph-1-predictor-664c5c98bd-8nr4d.18b14aea57996334 namespace: kserve-ci-e2e-test resourceVersion: "14515" uid: 9f763eca-2767-43c4-bb30-1c109e85336e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-747dc7bb84-jx2kx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:38Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-664c5c98bd-8nr4d namespace: kserve-ci-e2e-test resourceVersion: "14512" uid: efd1378d-8fc2-43b0-9a18-4cce7fac91f9 kind: Event lastTimestamp: "2026-05-20T14:01:38Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-20T14:01:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:38Z" name: isvc-sklearn-graph-1-predictor-664c5c98bd-8nr4d.18b14aea60341e70 namespace: kserve-ci-e2e-test resourceVersion: "14540" uid: 64df6b16-bd61-4562-9b63-8dd4a2ad46b8 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:39Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-664c5c98bd-8nr4d namespace: kserve-ci-e2e-test resourceVersion: "14520" uid: efd1378d-8fc2-43b0-9a18-4cce7fac91f9 kind: Event lastTimestamp: "2026-05-20T14:01:39Z" message: Add eth0 [10.133.0.18/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T14:01:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T14:01:39Z" name: isvc-sklearn-graph-1-predictor-664c5c98bd-8nr4d.18b14aea95469021 namespace: kserve-ci-e2e-test resourceVersion: "14636" uid: 746a0660-e0f0-4926-a74f-717875bcb86a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-664c5c98bd-8nr4d namespace: kserve-ci-e2e-test resourceVersion: "14512" uid: efd1378d-8fc2-43b0-9a18-4cce7fac91f9 kind: Event lastTimestamp: "2026-05-20T14:01:39Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0459e31fff147b1f9487117ac00a86221572ce2251d3068d36cf76ed2126ea4f" already present on machine metadata: creationTimestamp: "2026-05-20T14:01:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:39Z" name: isvc-sklearn-graph-1-predictor-664c5c98bd-8nr4d.18b14aea97433da0 namespace: kserve-ci-e2e-test resourceVersion: "14638" uid: 13abea38-ad93-4d37-9cc6-083d219a8efe reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-664c5c98bd-8nr4d namespace: kserve-ci-e2e-test resourceVersion: "14512" uid: efd1378d-8fc2-43b0-9a18-4cce7fac91f9 kind: Event lastTimestamp: "2026-05-20T14:01:39Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-20T14:01:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:39Z" name: isvc-sklearn-graph-1-predictor-664c5c98bd-8nr4d.18b14aea9a2d9c58 namespace: kserve-ci-e2e-test resourceVersion: "14641" uid: 83a2491d-6cc9-473f-86f9-178cb7241988 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-664c5c98bd-8nr4d namespace: kserve-ci-e2e-test resourceVersion: "14512" uid: efd1378d-8fc2-43b0-9a18-4cce7fac91f9 kind: Event lastTimestamp: "2026-05-20T14:01:39Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-20T14:01:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:39Z" name: isvc-sklearn-graph-1-predictor-664c5c98bd-8nr4d.18b14aea9aae9658 namespace: kserve-ci-e2e-test resourceVersion: "14642" uid: f3874b1d-62a7-45d1-8d0a-d5b290bbf158 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-664c5c98bd-8nr4d namespace: kserve-ci-e2e-test resourceVersion: "14512" uid: efd1378d-8fc2-43b0-9a18-4cce7fac91f9 kind: Event lastTimestamp: "2026-05-20T14:01:43Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1501" metadata: creationTimestamp: "2026-05-20T14:01:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:43Z" name: isvc-sklearn-graph-1-predictor-664c5c98bd-8nr4d.18b14aebb2b8968d namespace: kserve-ci-e2e-test resourceVersion: "14721" uid: 992d9b66-86ed-4200-8641-f5762f0edcf8 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-664c5c98bd-8nr4d namespace: kserve-ci-e2e-test resourceVersion: "14512" uid: efd1378d-8fc2-43b0-9a18-4cce7fac91f9 kind: Event lastTimestamp: "2026-05-20T14:01:55Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1501" in 11.858s (11.858s including waiting). Image size: 1560929697 bytes.' metadata: creationTimestamp: "2026-05-20T14:01:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:55Z" name: isvc-sklearn-graph-1-predictor-664c5c98bd-8nr4d.18b14aee758cf7e8 namespace: kserve-ci-e2e-test resourceVersion: "14837" uid: da4b954b-f63b-4a7f-8cd8-2c4c9c634284 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-664c5c98bd-8nr4d namespace: kserve-ci-e2e-test resourceVersion: "14512" uid: efd1378d-8fc2-43b0-9a18-4cce7fac91f9 kind: Event lastTimestamp: "2026-05-20T14:01:55Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T14:01:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:55Z" name: isvc-sklearn-graph-1-predictor-664c5c98bd-8nr4d.18b14aee7b4a06c8 namespace: kserve-ci-e2e-test resourceVersion: "14840" uid: 62a9b2f1-dbe2-4c38-803c-30563f887477 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-664c5c98bd-8nr4d namespace: kserve-ci-e2e-test resourceVersion: "14512" uid: efd1378d-8fc2-43b0-9a18-4cce7fac91f9 kind: Event lastTimestamp: "2026-05-20T14:01:55Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T14:01:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:55Z" name: isvc-sklearn-graph-1-predictor-664c5c98bd-8nr4d.18b14aee7bf9219f namespace: kserve-ci-e2e-test resourceVersion: "14845" uid: 56dc9154-a0fc-45b6-bf77-3a734a9c7d75 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-664c5c98bd-8nr4d namespace: kserve-ci-e2e-test resourceVersion: "14512" uid: efd1378d-8fc2-43b0-9a18-4cce7fac91f9 kind: Event lastTimestamp: "2026-05-20T14:01:55Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-05-20T14:01:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:55Z" name: isvc-sklearn-graph-1-predictor-664c5c98bd-8nr4d.18b14aee7c07a266 namespace: kserve-ci-e2e-test resourceVersion: "14846" uid: 2d5e9ec1-1d3f-4b52-8905-42298e809927 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-664c5c98bd-8nr4d namespace: kserve-ci-e2e-test resourceVersion: "14512" uid: efd1378d-8fc2-43b0-9a18-4cce7fac91f9 kind: Event lastTimestamp: "2026-05-20T14:01:58Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.606s (2.606s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-05-20T14:01:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:58Z" name: isvc-sklearn-graph-1-predictor-664c5c98bd-8nr4d.18b14aef1761289d namespace: kserve-ci-e2e-test resourceVersion: "14869" uid: 4aadd992-63a4-4db9-a49b-2f817fa56de2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-664c5c98bd-8nr4d namespace: kserve-ci-e2e-test resourceVersion: "14512" uid: efd1378d-8fc2-43b0-9a18-4cce7fac91f9 kind: Event lastTimestamp: "2026-05-20T14:01:58Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T14:01:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:58Z" name: isvc-sklearn-graph-1-predictor-664c5c98bd-8nr4d.18b14aef1b3899ca namespace: kserve-ci-e2e-test resourceVersion: "14874" uid: 95e6bcda-689f-40cb-95fc-3b19ad2bbf16 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-664c5c98bd-8nr4d namespace: kserve-ci-e2e-test resourceVersion: "14512" uid: efd1378d-8fc2-43b0-9a18-4cce7fac91f9 kind: Event lastTimestamp: "2026-05-20T14:01:58Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:01:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:58Z" name: isvc-sklearn-graph-1-predictor-664c5c98bd-8nr4d.18b14aef1bcaa081 namespace: kserve-ci-e2e-test resourceVersion: "14876" uid: 7f4a969e-7810-4a35-a220-5fccd9f9ee3b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-05-20T14:02:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-664c5c98bd-8nr4d namespace: kserve-ci-e2e-test resourceVersion: "14512" uid: efd1378d-8fc2-43b0-9a18-4cce7fac91f9 kind: Event lastTimestamp: "2026-05-20T14:03:56Z" message: 'Readiness probe failed: dial tcp 10.133.0.18:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T14:02:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:56Z" name: isvc-sklearn-graph-1-predictor-664c5c98bd-8nr4d.18b14aef70c334f5 namespace: kserve-ci-e2e-test resourceVersion: "16275" uid: 3ad3523c-cc62-491a-84e7-0969462b764c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-664c5c98bd-8nr4d namespace: kserve-ci-e2e-test resourceVersion: "14512" uid: efd1378d-8fc2-43b0-9a18-4cce7fac91f9 kind: Event lastTimestamp: "2026-05-20T14:03:48Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T14:03:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:48Z" name: isvc-sklearn-graph-1-predictor-664c5c98bd-8nr4d.18b14b08b57fbc55 namespace: kserve-ci-e2e-test resourceVersion: "16100" uid: 9ab47e5d-1772-41e9-9251-0b338d77bc9c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-664c5c98bd-8nr4d namespace: kserve-ci-e2e-test resourceVersion: "14512" uid: efd1378d-8fc2-43b0-9a18-4cce7fac91f9 kind: Event lastTimestamp: "2026-05-20T14:03:48Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:03:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:48Z" name: isvc-sklearn-graph-1-predictor-664c5c98bd-8nr4d.18b14b08b58065bc namespace: kserve-ci-e2e-test resourceVersion: "16104" uid: 2f61453e-0d45-41e6-b495-1f5a3d7156f6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T14:03:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-664c5c98bd-8nr4d namespace: kserve-ci-e2e-test resourceVersion: "14512" uid: efd1378d-8fc2-43b0-9a18-4cce7fac91f9 kind: Event lastTimestamp: "2026-05-20T14:03:56Z" message: 'Readiness probe failed: Get "https://10.133.0.18:8643/healthz": dial tcp 10.133.0.18:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-20T14:03:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:56Z" name: isvc-sklearn-graph-1-predictor-664c5c98bd-8nr4d.18b14b09490672f9 namespace: kserve-ci-e2e-test resourceVersion: "16273" uid: 1fe12815-33e8-4efe-ac65-7d246b1c1ff9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:38Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-664c5c98bd namespace: kserve-ci-e2e-test resourceVersion: "14498" uid: c60e1ec5-f1a1-4081-9236-7dc709cb2ac2 kind: Event lastTimestamp: "2026-05-20T14:01:38Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-664c5c98bd-8nr4d' metadata: creationTimestamp: "2026-05-20T14:01:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:01:38Z" name: isvc-sklearn-graph-1-predictor-664c5c98bd.18b14aea570c6409 namespace: kserve-ci-e2e-test resourceVersion: "14510" uid: ab09d353-8d34-42e7-b169-8c1d46226442 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:38Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14496" uid: c18eaa65-b2b4-4f46-b917-e535cbf2fc22 kind: Event lastTimestamp: "2026-05-20T14:01:38Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-664c5c98bd from 0 to 1 metadata: creationTimestamp: "2026-05-20T14:01:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:01:38Z" name: isvc-sklearn-graph-1-predictor.18b14aea50b83c94 namespace: kserve-ci-e2e-test resourceVersion: "14500" uid: f574a2d5-74aa-45f3-b672-a4db1e42ef1f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T14:01:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14508" uid: 0291d261-ce29-4ba0-b278-a5e7f4c2d15c kind: Event lastTimestamp: "2026-05-20T14:02:23Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:01:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:02:23Z" name: isvc-sklearn-graph-1-predictor.18b14aedd842b5d0 namespace: kserve-ci-e2e-test resourceVersion: "15086" uid: 3a5fbffd-bdc2-423e-994e-42eb3e72d5ac reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T14:01:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14508" uid: 0291d261-ce29-4ba0-b278-a5e7f4c2d15c kind: Event lastTimestamp: "2026-05-20T14:02:23Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:01:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:02:23Z" name: isvc-sklearn-graph-1-predictor.18b14aedd8cdce44 namespace: kserve-ci-e2e-test resourceVersion: "15087" uid: c2224ed6-b8d0-4a0d-aea8-7f5d39418456 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-20T14:02:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14805" uid: 0291d261-ce29-4ba0-b278-a5e7f4c2d15c kind: Event lastTimestamp: "2026-05-20T14:03:38Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:02:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:03:38Z" name: isvc-sklearn-graph-1-predictor.18b14af85715460d namespace: kserve-ci-e2e-test resourceVersion: "15939" uid: 5be53c1b-7dd3-435f-8b77-770fd8c27e4e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-20T14:02:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14805" uid: 0291d261-ce29-4ba0-b278-a5e7f4c2d15c kind: Event lastTimestamp: "2026-05-20T14:03:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:02:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:03:38Z" name: isvc-sklearn-graph-1-predictor.18b14af85789bd99 namespace: kserve-ci-e2e-test resourceVersion: "15940" uid: bbea5fb0-96f3-4509-a72b-3630de96327c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-05-20T14:01:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14480" uid: 2226b906-1c95-425d-8f72-e1ad6918411a kind: Event lastTimestamp: "2026-05-20T14:03:16Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T14:01:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:03:16Z" name: isvc-sklearn-graph-1.18b14aea570f17fa namespace: kserve-ci-e2e-test resourceVersion: "15573" uid: c84280a3-5bb5-4fd5-b176-49a7cc9c3b7f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "15570" uid: 2226b906-1c95-425d-8f72-e1ad6918411a kind: Event lastTimestamp: "2026-05-20T14:03:16Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-05-20T14:03:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:03:16Z" name: isvc-sklearn-graph-1.18b14b012abdd6dd namespace: kserve-ci-e2e-test resourceVersion: "15572" uid: a8942c16-33a1-4f69-a55f-803c6da6ad8c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T14:01:39.501489Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-58cc67cb8-c6bdg namespace: kserve-ci-e2e-test resourceVersion: "14645" uid: d35c4ca2-5c86-491f-8694-a7904ffd2bc8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-58cc67cb8-c6bdg to ip-10-0-137-148.ec2.internal metadata: creationTimestamp: "2026-05-20T14:01:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T14:01:39Z" name: isvc-sklearn-graph-2-predictor-58cc67cb8-c6bdg.18b14aeaa8b822c7 namespace: kserve-ci-e2e-test resourceVersion: "14649" uid: 6b6b8fa0-2e60-4dcb-bb0e-914e2dfc9837 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-747dc7bb84-jx2kx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:40Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-58cc67cb8-c6bdg namespace: kserve-ci-e2e-test resourceVersion: "14654" uid: d35c4ca2-5c86-491f-8694-a7904ffd2bc8 kind: Event lastTimestamp: "2026-05-20T14:01:40Z" message: Add eth0 [10.133.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T14:01:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T14:01:40Z" name: isvc-sklearn-graph-2-predictor-58cc67cb8-c6bdg.18b14aeae91c7cae namespace: kserve-ci-e2e-test resourceVersion: "14673" uid: e8b2d390-2986-486c-b050-4d87378f8fee reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-58cc67cb8-c6bdg namespace: kserve-ci-e2e-test resourceVersion: "14646" uid: d35c4ca2-5c86-491f-8694-a7904ffd2bc8 kind: Event lastTimestamp: "2026-05-20T14:01:40Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0459e31fff147b1f9487117ac00a86221572ce2251d3068d36cf76ed2126ea4f" already present on machine metadata: creationTimestamp: "2026-05-20T14:01:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:40Z" name: isvc-sklearn-graph-2-predictor-58cc67cb8-c6bdg.18b14aeaebf9cadb namespace: kserve-ci-e2e-test resourceVersion: "14677" uid: 0183d264-afc2-4ce6-abe5-63984f6949ee reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-58cc67cb8-c6bdg namespace: kserve-ci-e2e-test resourceVersion: "14646" uid: d35c4ca2-5c86-491f-8694-a7904ffd2bc8 kind: Event lastTimestamp: "2026-05-20T14:01:40Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-20T14:01:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:40Z" name: isvc-sklearn-graph-2-predictor-58cc67cb8-c6bdg.18b14aeaefdb7551 namespace: kserve-ci-e2e-test resourceVersion: "14679" uid: eb3f930c-4db5-4c6b-9589-bac2b6837d47 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-58cc67cb8-c6bdg namespace: kserve-ci-e2e-test resourceVersion: "14646" uid: d35c4ca2-5c86-491f-8694-a7904ffd2bc8 kind: Event lastTimestamp: "2026-05-20T14:01:40Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-20T14:01:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:40Z" name: isvc-sklearn-graph-2-predictor-58cc67cb8-c6bdg.18b14aeaf08bbd33 namespace: kserve-ci-e2e-test resourceVersion: "14680" uid: 7803c385-33da-4021-bb4e-6fc062fe3ed4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-58cc67cb8-c6bdg namespace: kserve-ci-e2e-test resourceVersion: "14646" uid: d35c4ca2-5c86-491f-8694-a7904ffd2bc8 kind: Event lastTimestamp: "2026-05-20T14:01:48Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1501" metadata: creationTimestamp: "2026-05-20T14:01:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:48Z" name: isvc-sklearn-graph-2-predictor-58cc67cb8-c6bdg.18b14aecddd721a7 namespace: kserve-ci-e2e-test resourceVersion: "14758" uid: aaa6f946-1bf0-43ea-9303-ec8ce97c02c4 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-58cc67cb8-c6bdg namespace: kserve-ci-e2e-test resourceVersion: "14646" uid: d35c4ca2-5c86-491f-8694-a7904ffd2bc8 kind: Event lastTimestamp: "2026-05-20T14:01:55Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1501" in 6.84s (6.84s including waiting). Image size: 1560929697 bytes.' metadata: creationTimestamp: "2026-05-20T14:01:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:55Z" name: isvc-sklearn-graph-2-predictor-58cc67cb8-c6bdg.18b14aee7592063d namespace: kserve-ci-e2e-test resourceVersion: "14838" uid: 6c2a1f83-8845-4d5f-a390-763580c37ad0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-58cc67cb8-c6bdg namespace: kserve-ci-e2e-test resourceVersion: "14646" uid: d35c4ca2-5c86-491f-8694-a7904ffd2bc8 kind: Event lastTimestamp: "2026-05-20T14:01:55Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T14:01:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:55Z" name: isvc-sklearn-graph-2-predictor-58cc67cb8-c6bdg.18b14aee7b54c2a9 namespace: kserve-ci-e2e-test resourceVersion: "14841" uid: 41bd46b3-915c-4984-9675-d3b255742624 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-58cc67cb8-c6bdg namespace: kserve-ci-e2e-test resourceVersion: "14646" uid: d35c4ca2-5c86-491f-8694-a7904ffd2bc8 kind: Event lastTimestamp: "2026-05-20T14:01:55Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T14:01:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:55Z" name: isvc-sklearn-graph-2-predictor-58cc67cb8-c6bdg.18b14aee7bde5fe7 namespace: kserve-ci-e2e-test resourceVersion: "14843" uid: 5c794827-c4e3-420f-96f4-d33f3fd8e581 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-58cc67cb8-c6bdg namespace: kserve-ci-e2e-test resourceVersion: "14646" uid: d35c4ca2-5c86-491f-8694-a7904ffd2bc8 kind: Event lastTimestamp: "2026-05-20T14:01:55Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-05-20T14:01:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:55Z" name: isvc-sklearn-graph-2-predictor-58cc67cb8-c6bdg.18b14aee7becd426 namespace: kserve-ci-e2e-test resourceVersion: "14844" uid: 5acc701f-5d21-49b2-a4be-dc162658d328 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-58cc67cb8-c6bdg namespace: kserve-ci-e2e-test resourceVersion: "14646" uid: d35c4ca2-5c86-491f-8694-a7904ffd2bc8 kind: Event lastTimestamp: "2026-05-20T14:01:58Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.595s (2.595s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-05-20T14:01:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:58Z" name: isvc-sklearn-graph-2-predictor-58cc67cb8-c6bdg.18b14aef16a843bf namespace: kserve-ci-e2e-test resourceVersion: "14868" uid: bcb92a64-6992-45a9-8d0a-c43479e8fbc3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-58cc67cb8-c6bdg namespace: kserve-ci-e2e-test resourceVersion: "14646" uid: d35c4ca2-5c86-491f-8694-a7904ffd2bc8 kind: Event lastTimestamp: "2026-05-20T14:01:58Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T14:01:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:58Z" name: isvc-sklearn-graph-2-predictor-58cc67cb8-c6bdg.18b14aef1b1941aa namespace: kserve-ci-e2e-test resourceVersion: "14873" uid: 9e094a21-2e94-4318-9a8b-7f63933865ac reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-58cc67cb8-c6bdg namespace: kserve-ci-e2e-test resourceVersion: "14646" uid: d35c4ca2-5c86-491f-8694-a7904ffd2bc8 kind: Event lastTimestamp: "2026-05-20T14:01:58Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:01:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:58Z" name: isvc-sklearn-graph-2-predictor-58cc67cb8-c6bdg.18b14aef1b96c071 namespace: kserve-ci-e2e-test resourceVersion: "14875" uid: ccfd6774-9e06-435c-994f-0a64cd8b0883 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-05-20T14:01:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-58cc67cb8-c6bdg namespace: kserve-ci-e2e-test resourceVersion: "14646" uid: d35c4ca2-5c86-491f-8694-a7904ffd2bc8 kind: Event lastTimestamp: "2026-05-20T14:03:06Z" message: 'Readiness probe failed: dial tcp 10.133.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T14:01:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:06Z" name: isvc-sklearn-graph-2-predictor-58cc67cb8-c6bdg.18b14aef34f4c844 namespace: kserve-ci-e2e-test resourceVersion: "15449" uid: d077850c-3cdd-4d37-94e0-162ff7cf3584 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-58cc67cb8-c6bdg namespace: kserve-ci-e2e-test resourceVersion: "14646" uid: d35c4ca2-5c86-491f-8694-a7904ffd2bc8 kind: Event lastTimestamp: "2026-05-20T14:03:48Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T14:03:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:48Z" name: isvc-sklearn-graph-2-predictor-58cc67cb8-c6bdg.18b14b08bec92eb6 namespace: kserve-ci-e2e-test resourceVersion: "16171" uid: 22f0093e-855f-4b79-a8a5-e84d63fd64e9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-58cc67cb8-c6bdg namespace: kserve-ci-e2e-test resourceVersion: "14646" uid: d35c4ca2-5c86-491f-8694-a7904ffd2bc8 kind: Event lastTimestamp: "2026-05-20T14:03:48Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:03:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:48Z" name: isvc-sklearn-graph-2-predictor-58cc67cb8-c6bdg.18b14b08bec9e12f namespace: kserve-ci-e2e-test resourceVersion: "16172" uid: 795d0bb2-af60-4ba9-a788-ab1dd87db3a8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-58cc67cb8-c6bdg namespace: kserve-ci-e2e-test resourceVersion: "14646" uid: d35c4ca2-5c86-491f-8694-a7904ffd2bc8 kind: Event lastTimestamp: "2026-05-20T14:03:50Z" message: 'Readiness probe failed: Get "https://10.133.0.21:8643/healthz": dial tcp 10.133.0.21:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-20T14:03:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:50Z" name: isvc-sklearn-graph-2-predictor-58cc67cb8-c6bdg.18b14b090d3eb475 namespace: kserve-ci-e2e-test resourceVersion: "16213" uid: 62129886-e156-46b3-8e6d-b04ab37dfe87 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-58cc67cb8 namespace: kserve-ci-e2e-test resourceVersion: "14620" uid: 02d71b03-fcd5-42a3-ab58-75a18b1b3a2c kind: Event lastTimestamp: "2026-05-20T14:01:39Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-58cc67cb8-c6bdg' metadata: creationTimestamp: "2026-05-20T14:01:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:01:39Z" name: isvc-sklearn-graph-2-predictor-58cc67cb8.18b14aeaa81dc84d namespace: kserve-ci-e2e-test resourceVersion: "14647" uid: 9100fb80-91f0-485b-99dc-5a9d9fc03140 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:38Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14618" uid: eecb474f-5f33-41eb-9c76-3fb3f5716674 kind: Event lastTimestamp: "2026-05-20T14:01:38Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-58cc67cb8 from 0 to 1 metadata: creationTimestamp: "2026-05-20T14:01:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:01:38Z" name: isvc-sklearn-graph-2-predictor.18b14aea88e02a9b namespace: kserve-ci-e2e-test resourceVersion: "14623" uid: 2db85883-f7f5-4537-a256-00cfd9253f59 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T14:01:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14621" uid: ec99da44-c424-4146-99b4-2aa24faa9b05 kind: Event lastTimestamp: "2026-05-20T14:02:24Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:01:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:02:24Z" name: isvc-sklearn-graph-2-predictor.18b14aee07e258d9 namespace: kserve-ci-e2e-test resourceVersion: "15100" uid: 53111510-9989-4ddd-936d-580c70aef9f3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T14:01:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14621" uid: ec99da44-c424-4146-99b4-2aa24faa9b05 kind: Event lastTimestamp: "2026-05-20T14:02:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:01:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:02:24Z" name: isvc-sklearn-graph-2-predictor.18b14aee0861370e namespace: kserve-ci-e2e-test resourceVersion: "15101" uid: 755083a8-6112-400c-be22-fcbf58dcd75f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-20T14:02:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14824" uid: ec99da44-c424-4146-99b4-2aa24faa9b05 kind: Event lastTimestamp: "2026-05-20T14:03:39Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:02:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:03:39Z" name: isvc-sklearn-graph-2-predictor.18b14af885e21203 namespace: kserve-ci-e2e-test resourceVersion: "15951" uid: a5af1971-5928-40b3-ae72-5a6474faff30 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-20T14:02:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14824" uid: ec99da44-c424-4146-99b4-2aa24faa9b05 kind: Event lastTimestamp: "2026-05-20T14:03:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:02:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:03:39Z" name: isvc-sklearn-graph-2-predictor.18b14af8865b285e namespace: kserve-ci-e2e-test resourceVersion: "15952" uid: 7d400018-c424-489b-9284-47c258ae2b56 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-05-20T14:01:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14594" uid: f1fed412-5267-4fac-baf1-a56119a0475b kind: Event lastTimestamp: "2026-05-20T14:03:17Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T14:01:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:03:17Z" name: isvc-sklearn-graph-2.18b14aea88f4662f namespace: kserve-ci-e2e-test resourceVersion: "15588" uid: 924b9485-1bf1-45a9-9866-d70f34605c02 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "15586" uid: f1fed412-5267-4fac-baf1-a56119a0475b kind: Event lastTimestamp: "2026-05-20T14:03:16Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-05-20T14:03:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:03:16Z" name: isvc-sklearn-graph-2.18b14b0159920b9a namespace: kserve-ci-e2e-test resourceVersion: "15587" uid: 8ea0a22a-27c2-495a-9646-1224cc41e00c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T14:01:38.696740Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-xhv8r namespace: kserve-ci-e2e-test resourceVersion: "14598" uid: 3dd098d1-0850-4e94-8fba-0197f4b1f733 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-669d8d6456-xhv8r to ip-10-0-137-148.ec2.internal metadata: creationTimestamp: "2026-05-20T14:01:38Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T14:01:38Z" name: isvc-xgboost-graph-predictor-669d8d6456-xhv8r.18b14aea78c0a48e namespace: kserve-ci-e2e-test resourceVersion: "14602" uid: 3aa1a946-4ce2-422a-9454-4526310964ed reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-747dc7bb84-jx2kx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:39Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-xhv8r namespace: kserve-ci-e2e-test resourceVersion: "14610" uid: 3dd098d1-0850-4e94-8fba-0197f4b1f733 kind: Event lastTimestamp: "2026-05-20T14:01:39Z" message: Add eth0 [10.133.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T14:01:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T14:01:39Z" name: isvc-xgboost-graph-predictor-669d8d6456-xhv8r.18b14aea9260889e namespace: kserve-ci-e2e-test resourceVersion: "14633" uid: b1eae4d2-e135-419b-b8b2-960ec38565c5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-xhv8r namespace: kserve-ci-e2e-test resourceVersion: "14600" uid: 3dd098d1-0850-4e94-8fba-0197f4b1f733 kind: Event lastTimestamp: "2026-05-20T14:01:39Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0459e31fff147b1f9487117ac00a86221572ce2251d3068d36cf76ed2126ea4f" already present on machine metadata: creationTimestamp: "2026-05-20T14:01:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:39Z" name: isvc-xgboost-graph-predictor-669d8d6456-xhv8r.18b14aea94605fdd namespace: kserve-ci-e2e-test resourceVersion: "14635" uid: 68bcc8f5-31ab-497e-8f4a-1b4c2b2cdf61 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-xhv8r namespace: kserve-ci-e2e-test resourceVersion: "14600" uid: 3dd098d1-0850-4e94-8fba-0197f4b1f733 kind: Event lastTimestamp: "2026-05-20T14:01:39Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-20T14:01:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:39Z" name: isvc-xgboost-graph-predictor-669d8d6456-xhv8r.18b14aea978782ff namespace: kserve-ci-e2e-test resourceVersion: "14639" uid: 4efe964b-23aa-4d08-8342-75fe63e7cb07 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-xhv8r namespace: kserve-ci-e2e-test resourceVersion: "14600" uid: 3dd098d1-0850-4e94-8fba-0197f4b1f733 kind: Event lastTimestamp: "2026-05-20T14:01:39Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-20T14:01:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:39Z" name: isvc-xgboost-graph-predictor-669d8d6456-xhv8r.18b14aea98062d4f namespace: kserve-ci-e2e-test resourceVersion: "14640" uid: 8a3421bb-f31e-4752-8cc7-0164206998df reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-xhv8r namespace: kserve-ci-e2e-test resourceVersion: "14600" uid: 3dd098d1-0850-4e94-8fba-0197f4b1f733 kind: Event lastTimestamp: "2026-05-20T14:01:43Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-05-20T14:01:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:43Z" name: isvc-xgboost-graph-predictor-669d8d6456-xhv8r.18b14aebb2a8ab42 namespace: kserve-ci-e2e-test resourceVersion: "14720" uid: a73ac21e-bfc0-4584-83f3-3b13301bfee6 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:02:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-xhv8r namespace: kserve-ci-e2e-test resourceVersion: "14600" uid: 3dd098d1-0850-4e94-8fba-0197f4b1f733 kind: Event lastTimestamp: "2026-05-20T14:02:11Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 27.172s (27.172s including waiting). Image size: 1306436347 bytes.' metadata: creationTimestamp: "2026-05-20T14:02:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:02:11Z" name: isvc-xgboost-graph-predictor-669d8d6456-xhv8r.18b14af20643f219 namespace: kserve-ci-e2e-test resourceVersion: "14994" uid: 528e57c4-b0f1-462a-8491-344ff4b8c3e2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:02:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-xhv8r namespace: kserve-ci-e2e-test resourceVersion: "14600" uid: 3dd098d1-0850-4e94-8fba-0197f4b1f733 kind: Event lastTimestamp: "2026-05-20T14:02:11Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T14:02:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:02:11Z" name: isvc-xgboost-graph-predictor-669d8d6456-xhv8r.18b14af20a84d6fb namespace: kserve-ci-e2e-test resourceVersion: "14995" uid: 460c510f-b886-407a-ad68-2331aef44400 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:02:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-xhv8r namespace: kserve-ci-e2e-test resourceVersion: "14600" uid: 3dd098d1-0850-4e94-8fba-0197f4b1f733 kind: Event lastTimestamp: "2026-05-20T14:02:11Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T14:02:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:02:11Z" name: isvc-xgboost-graph-predictor-669d8d6456-xhv8r.18b14af20ae5bfd0 namespace: kserve-ci-e2e-test resourceVersion: "14996" uid: e55df187-e579-4334-884b-70e78f084b2d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:02:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-xhv8r namespace: kserve-ci-e2e-test resourceVersion: "14600" uid: 3dd098d1-0850-4e94-8fba-0197f4b1f733 kind: Event lastTimestamp: "2026-05-20T14:02:11Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-20T14:02:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:02:11Z" name: isvc-xgboost-graph-predictor-669d8d6456-xhv8r.18b14af20afe1815 namespace: kserve-ci-e2e-test resourceVersion: "14997" uid: c6ce053c-4c9e-4044-b422-0237801658d0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:02:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-xhv8r namespace: kserve-ci-e2e-test resourceVersion: "14600" uid: 3dd098d1-0850-4e94-8fba-0197f4b1f733 kind: Event lastTimestamp: "2026-05-20T14:02:11Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T14:02:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:02:11Z" name: isvc-xgboost-graph-predictor-669d8d6456-xhv8r.18b14af20e95137d namespace: kserve-ci-e2e-test resourceVersion: "14998" uid: b96c85ce-ca12-42d3-a7f1-87d396a20d81 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:02:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-xhv8r namespace: kserve-ci-e2e-test resourceVersion: "14600" uid: 3dd098d1-0850-4e94-8fba-0197f4b1f733 kind: Event lastTimestamp: "2026-05-20T14:02:11Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:02:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:02:11Z" name: isvc-xgboost-graph-predictor-669d8d6456-xhv8r.18b14af20ef66f11 namespace: kserve-ci-e2e-test resourceVersion: "14999" uid: 94dc8271-7e18-4bc9-b31f-7a6a2a18d630 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-05-20T14:02:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-xhv8r namespace: kserve-ci-e2e-test resourceVersion: "14600" uid: 3dd098d1-0850-4e94-8fba-0197f4b1f733 kind: Event lastTimestamp: "2026-05-20T14:03:49Z" message: 'Readiness probe failed: dial tcp 10.133.0.20:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T14:02:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:49Z" name: isvc-xgboost-graph-predictor-669d8d6456-xhv8r.18b14af27aa622ba namespace: kserve-ci-e2e-test resourceVersion: "16184" uid: 8d0092db-068d-4880-84b2-7e8817835ede reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-xhv8r namespace: kserve-ci-e2e-test resourceVersion: "14600" uid: 3dd098d1-0850-4e94-8fba-0197f4b1f733 kind: Event lastTimestamp: "2026-05-20T14:03:48Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T14:03:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:48Z" name: isvc-xgboost-graph-predictor-669d8d6456-xhv8r.18b14b08ba9f525b namespace: kserve-ci-e2e-test resourceVersion: "16139" uid: 2bbac934-7452-4dbe-8b03-d8735da4cfe6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-xhv8r namespace: kserve-ci-e2e-test resourceVersion: "14600" uid: 3dd098d1-0850-4e94-8fba-0197f4b1f733 kind: Event lastTimestamp: "2026-05-20T14:03:48Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:03:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:48Z" name: isvc-xgboost-graph-predictor-669d8d6456-xhv8r.18b14b08ba9fe2e5 namespace: kserve-ci-e2e-test resourceVersion: "16141" uid: a0eff3cd-7256-4076-a71f-de5ad9e2a9e8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-xhv8r namespace: kserve-ci-e2e-test resourceVersion: "14600" uid: 3dd098d1-0850-4e94-8fba-0197f4b1f733 kind: Event lastTimestamp: "2026-05-20T14:03:49Z" message: 'Readiness probe failed: Get "https://10.133.0.20:8643/healthz": dial tcp 10.133.0.20:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-20T14:03:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:49Z" name: isvc-xgboost-graph-predictor-669d8d6456-xhv8r.18b14b08d4d4f490 namespace: kserve-ci-e2e-test resourceVersion: "16182" uid: 47a0badc-f8a7-4a98-8899-4f43d2d5708d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:38Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-669d8d6456 namespace: kserve-ci-e2e-test resourceVersion: "14584" uid: a32d660d-d56d-4992-abca-4f68be41ef1b kind: Event lastTimestamp: "2026-05-20T14:01:38Z" message: 'Created pod: isvc-xgboost-graph-predictor-669d8d6456-xhv8r' metadata: creationTimestamp: "2026-05-20T14:01:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:01:38Z" name: isvc-xgboost-graph-predictor-669d8d6456.18b14aea783d82c1 namespace: kserve-ci-e2e-test resourceVersion: "14599" uid: 039668fd-74e1-425c-862b-c44a15f6198c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:38Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14580" uid: 860ed6aa-9f88-4b73-b96c-a2769dc5a00c kind: Event lastTimestamp: "2026-05-20T14:01:38Z" message: Scaled up replica set isvc-xgboost-graph-predictor-669d8d6456 from 0 to 1 metadata: creationTimestamp: "2026-05-20T14:01:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:01:38Z" name: isvc-xgboost-graph-predictor.18b14aea6f0b82b5 namespace: kserve-ci-e2e-test resourceVersion: "14585" uid: f6160565-53eb-4bc4-ac1d-c7f201090c8e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-20T14:01:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14583" uid: d4542168-e03e-4b0d-ba57-f514c54f7828 kind: Event lastTimestamp: "2026-05-20T14:02:53Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:01:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:02:53Z" name: isvc-xgboost-graph-predictor.18b14aededd97d7e namespace: kserve-ci-e2e-test resourceVersion: "15345" uid: f960f93f-daab-4908-a8f3-2e65c448195b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-20T14:01:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14583" uid: d4542168-e03e-4b0d-ba57-f514c54f7828 kind: Event lastTimestamp: "2026-05-20T14:02:53Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:01:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:02:53Z" name: isvc-xgboost-graph-predictor.18b14aedee4f6393 namespace: kserve-ci-e2e-test resourceVersion: "15346" uid: 82fd0d75-1dbc-459a-860f-0bb27274fb7c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T14:03:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14816" uid: d4542168-e03e-4b0d-ba57-f514c54f7828 kind: Event lastTimestamp: "2026-05-20T14:03:38Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:03:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:03:38Z" name: isvc-xgboost-graph-predictor.18b14aff69bebbd6 namespace: kserve-ci-e2e-test resourceVersion: "15947" uid: 62573a70-020d-4561-ba59-e1d4791fb386 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T14:03:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14816" uid: d4542168-e03e-4b0d-ba57-f514c54f7828 kind: Event lastTimestamp: "2026-05-20T14:03:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:03:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:03:38Z" name: isvc-xgboost-graph-predictor.18b14aff6a355735 namespace: kserve-ci-e2e-test resourceVersion: "15948" uid: 3a5398b0-be3d-4467-ad4b-6fbef678c4d4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-05-20T14:01:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14566" uid: afedf5fd-5920-4f30-b319-55511a503d0c kind: Event lastTimestamp: "2026-05-20T14:03:20Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T14:01:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:03:20Z" name: isvc-xgboost-graph.18b14aea6f08549c namespace: kserve-ci-e2e-test resourceVersion: "15764" uid: d4b66332-a1d5-4703-a3f9-d2119edb7b39 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15752" uid: afedf5fd-5920-4f30-b319-55511a503d0c kind: Event lastTimestamp: "2026-05-20T14:03:19Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-05-20T14:03:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:03:20Z" name: isvc-xgboost-graph.18b14b020a5b1359 namespace: kserve-ci-e2e-test resourceVersion: "15758" uid: d345fdf1-53de-4c58-9e93-4686fac3a63b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T14:03:38.218424Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-d669cd755-txm6r namespace: kserve-ci-e2e-test resourceVersion: "15923" uid: 324d13b5-bc33-4b21-a680-3ff3788677a1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-d669cd755-txm6r to ip-10-0-136-75.ec2.internal metadata: creationTimestamp: "2026-05-20T14:03:38Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T14:03:38Z" name: model-chainer-d669cd755-txm6r.18b14b064cccc91f namespace: kserve-ci-e2e-test resourceVersion: "15927" uid: d4cc7e72-8cdb-4478-a068-c8540e617500 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-747dc7bb84-jx2kx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:38Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-d669cd755-txm6r namespace: kserve-ci-e2e-test resourceVersion: "15924" uid: 324d13b5-bc33-4b21-a680-3ff3788677a1 kind: Event lastTimestamp: "2026-05-20T14:03:38Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-serving-cert" not found' metadata: creationTimestamp: "2026-05-20T14:03:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:38Z" name: model-chainer-d669cd755-txm6r.18b14b06585f3883 namespace: kserve-ci-e2e-test resourceVersion: "15942" uid: 4275e98c-5816-4618-ae1f-511be5fb6dab reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:39Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-d669cd755-txm6r namespace: kserve-ci-e2e-test resourceVersion: "15932" uid: 324d13b5-bc33-4b21-a680-3ff3788677a1 kind: Event lastTimestamp: "2026-05-20T14:03:39Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T14:03:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T14:03:39Z" name: model-chainer-d669cd755-txm6r.18b14b068a732637 namespace: kserve-ci-e2e-test resourceVersion: "15954" uid: 5c160311-46fe-49e1-b617-b79b319f8703 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-d669cd755-txm6r namespace: kserve-ci-e2e-test resourceVersion: "15924" uid: 324d13b5-bc33-4b21-a680-3ff3788677a1 kind: Event lastTimestamp: "2026-05-20T14:03:39Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:04926a9d7185ff179745ae32ed77ae962904508dbcb7541e9bf600392af6f503" already present on machine metadata: creationTimestamp: "2026-05-20T14:03:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:39Z" name: model-chainer-d669cd755-txm6r.18b14b068c120dd2 namespace: kserve-ci-e2e-test resourceVersion: "15956" uid: 8607dc0d-d072-4add-9c1f-7956d34e01bc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-d669cd755-txm6r namespace: kserve-ci-e2e-test resourceVersion: "15924" uid: 324d13b5-bc33-4b21-a680-3ff3788677a1 kind: Event lastTimestamp: "2026-05-20T14:03:39Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-05-20T14:03:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:39Z" name: model-chainer-d669cd755-txm6r.18b14b068eccc8f5 namespace: kserve-ci-e2e-test resourceVersion: "15957" uid: 71adf3c7-aa7b-447c-9086-26d9ecf806a3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-d669cd755-txm6r namespace: kserve-ci-e2e-test resourceVersion: "15924" uid: 324d13b5-bc33-4b21-a680-3ff3788677a1 kind: Event lastTimestamp: "2026-05-20T14:03:39Z" message: Started container model-chainer metadata: creationTimestamp: "2026-05-20T14:03:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:39Z" name: model-chainer-d669cd755-txm6r.18b14b068f3a2c30 namespace: kserve-ci-e2e-test resourceVersion: "15958" uid: 639ee785-5eae-4a36-8586-43d795ce8cd0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-d669cd755-txm6r namespace: kserve-ci-e2e-test resourceVersion: "15924" uid: 324d13b5-bc33-4b21-a680-3ff3788677a1 kind: Event lastTimestamp: "2026-05-20T14:03:48Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-05-20T14:03:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:48Z" name: model-chainer-d669cd755-txm6r.18b14b08a95d4afa namespace: kserve-ci-e2e-test resourceVersion: "16059" uid: fe5cbc22-83b6-411a-b979-107f44932907 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-20T14:03:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-d669cd755-txm6r namespace: kserve-ci-e2e-test resourceVersion: "15924" uid: 324d13b5-bc33-4b21-a680-3ff3788677a1 kind: Event lastTimestamp: "2026-05-20T14:04:16Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-20T14:03:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:04:16Z" name: model-chainer-d669cd755-txm6r.18b14b09498def6a namespace: kserve-ci-e2e-test resourceVersion: "16457" uid: 413972b2-95fd-4745-88e8-bec351985be4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:38Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-d669cd755 namespace: kserve-ci-e2e-test resourceVersion: "15919" uid: e9e1f1c2-9225-49e7-9edb-eab69b092cdc kind: Event lastTimestamp: "2026-05-20T14:03:38Z" message: 'Created pod: model-chainer-d669cd755-txm6r' metadata: creationTimestamp: "2026-05-20T14:03:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:03:38Z" name: model-chainer-d669cd755.18b14b064c6fd2f5 namespace: kserve-ci-e2e-test resourceVersion: "15925" uid: e77975e7-4703-406c-8190-aab39f57c710 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:38Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15917" uid: bc9473c6-fd4f-45c5-9862-cfcfc1a709f1 kind: Event lastTimestamp: "2026-05-20T14:03:38Z" message: Scaled up replica set model-chainer-d669cd755 from 0 to 1 metadata: creationTimestamp: "2026-05-20T14:03:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:03:38Z" name: model-chainer.18b14b064b894f69 namespace: kserve-ci-e2e-test resourceVersion: "15921" uid: 0a24ca4f-ba3d-4423-92b3-c721086bada9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:38Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15935" uid: 4aed68a1-cc43-42ee-818e-f0a408430382 kind: Event lastTimestamp: "2026-05-20T14:03:38Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-05-20T14:03:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:03:38Z" name: model-chainer.18b14b0651bb0179 namespace: kserve-ci-e2e-test resourceVersion: "15937" uid: d14ee342-9d37-4443-ab54-2a4484ecfadb reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T14:14:03.430403Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-14a90-6db7cfb9f9-9d8xf namespace: kserve-ci-e2e-test resourceVersion: "21185" uid: faae5ee3-1a59-4460-86dd-a79d893701c1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-14a90-6db7cfb9f9-9d8xf to ip-10-0-136-75.ec2.internal metadata: creationTimestamp: "2026-05-20T14:14:03Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T14:14:03Z" name: sequence-graph-14a90-6db7cfb9f9-9d8xf.18b14b97de567c6c namespace: kserve-ci-e2e-test resourceVersion: "21189" uid: 06083afc-7b90-47bc-8ee3-b4d969668939 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-747dc7bb84-jx2kx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:14:03Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-14a90-6db7cfb9f9-9d8xf namespace: kserve-ci-e2e-test resourceVersion: "21186" uid: faae5ee3-1a59-4460-86dd-a79d893701c1 kind: Event lastTimestamp: "2026-05-20T14:14:03Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-14a90-serving-cert" not found' metadata: creationTimestamp: "2026-05-20T14:14:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:14:03Z" name: sequence-graph-14a90-6db7cfb9f9-9d8xf.18b14b97e8faeae3 namespace: kserve-ci-e2e-test resourceVersion: "21204" uid: 7f602653-e7b4-4ff3-b3f0-d9e77fe56d61 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:14:04Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-14a90-6db7cfb9f9-9d8xf namespace: kserve-ci-e2e-test resourceVersion: "21195" uid: faae5ee3-1a59-4460-86dd-a79d893701c1 kind: Event lastTimestamp: "2026-05-20T14:14:04Z" message: Add eth0 [10.134.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T14:14:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T14:14:04Z" name: sequence-graph-14a90-6db7cfb9f9-9d8xf.18b14b981be63fad namespace: kserve-ci-e2e-test resourceVersion: "21215" uid: 72ef2f2f-0a97-4104-bc22-a43ba4b1a58b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:14:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-14a90} kind: Pod name: sequence-graph-14a90-6db7cfb9f9-9d8xf namespace: kserve-ci-e2e-test resourceVersion: "21186" uid: faae5ee3-1a59-4460-86dd-a79d893701c1 kind: Event lastTimestamp: "2026-05-20T14:14:04Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:04926a9d7185ff179745ae32ed77ae962904508dbcb7541e9bf600392af6f503" already present on machine metadata: creationTimestamp: "2026-05-20T14:14:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:14:04Z" name: sequence-graph-14a90-6db7cfb9f9-9d8xf.18b14b981d4194f2 namespace: kserve-ci-e2e-test resourceVersion: "21217" uid: 8572fdd8-5822-4b1a-ae99-7a1ececcc27c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:14:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-14a90} kind: Pod name: sequence-graph-14a90-6db7cfb9f9-9d8xf namespace: kserve-ci-e2e-test resourceVersion: "21186" uid: faae5ee3-1a59-4460-86dd-a79d893701c1 kind: Event lastTimestamp: "2026-05-20T14:14:04Z" message: 'Created container: sequence-graph-14a90' metadata: creationTimestamp: "2026-05-20T14:14:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:14:04Z" name: sequence-graph-14a90-6db7cfb9f9-9d8xf.18b14b98209f7d4f namespace: kserve-ci-e2e-test resourceVersion: "21218" uid: 22ddd106-d145-48fe-9232-f465127fcf7f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:14:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-14a90} kind: Pod name: sequence-graph-14a90-6db7cfb9f9-9d8xf namespace: kserve-ci-e2e-test resourceVersion: "21186" uid: faae5ee3-1a59-4460-86dd-a79d893701c1 kind: Event lastTimestamp: "2026-05-20T14:14:04Z" message: Started container sequence-graph-14a90 metadata: creationTimestamp: "2026-05-20T14:14:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:14:04Z" name: sequence-graph-14a90-6db7cfb9f9-9d8xf.18b14b982118f042 namespace: kserve-ci-e2e-test resourceVersion: "21219" uid: 6c5d0478-295d-4ac2-b9ac-d4800085a9c9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:14:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-14a90} kind: Pod name: sequence-graph-14a90-6db7cfb9f9-9d8xf namespace: kserve-ci-e2e-test resourceVersion: "21186" uid: faae5ee3-1a59-4460-86dd-a79d893701c1 kind: Event lastTimestamp: "2026-05-20T14:14:13Z" message: Stopping container sequence-graph-14a90 metadata: creationTimestamp: "2026-05-20T14:14:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:14:13Z" name: sequence-graph-14a90-6db7cfb9f9-9d8xf.18b14b9a38ba7a67 namespace: kserve-ci-e2e-test resourceVersion: "21318" uid: fb55eca5-7b7c-49df-81c5-c313dd90089f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-20T14:14:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-14a90} kind: Pod name: sequence-graph-14a90-6db7cfb9f9-9d8xf namespace: kserve-ci-e2e-test resourceVersion: "21186" uid: faae5ee3-1a59-4460-86dd-a79d893701c1 kind: Event lastTimestamp: "2026-05-20T14:14:41Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-20T14:14:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:14:41Z" name: sequence-graph-14a90-6db7cfb9f9-9d8xf.18b14b9adb134fd5 namespace: kserve-ci-e2e-test resourceVersion: "21686" uid: 8745d2e4-99f9-4afc-98f7-9b5441ecad73 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:14:03Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-14a90-6db7cfb9f9 namespace: kserve-ci-e2e-test resourceVersion: "21182" uid: 5fa8d14a-eb4b-484f-8634-7c15981c8548 kind: Event lastTimestamp: "2026-05-20T14:14:03Z" message: 'Created pod: sequence-graph-14a90-6db7cfb9f9-9d8xf' metadata: creationTimestamp: "2026-05-20T14:14:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:14:03Z" name: sequence-graph-14a90-6db7cfb9f9.18b14b97dde4987b namespace: kserve-ci-e2e-test resourceVersion: "21187" uid: 676a59e0-e1b5-4280-8cb4-b1f8fbba071a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:14:03Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-14a90 namespace: kserve-ci-e2e-test resourceVersion: "21180" uid: 8bd53cfa-d12e-49af-bfb0-6226b1d57122 kind: Event lastTimestamp: "2026-05-20T14:14:03Z" message: Scaled up replica set sequence-graph-14a90-6db7cfb9f9 from 0 to 1 metadata: creationTimestamp: "2026-05-20T14:14:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:14:03Z" name: sequence-graph-14a90.18b14b97dd1e3aa4 namespace: kserve-ci-e2e-test resourceVersion: "21183" uid: ef666d0d-7a72-4267-9513-cc1b5752b701 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:14:03Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-14a90 namespace: kserve-ci-e2e-test resourceVersion: "21200" uid: b8de5b3a-5214-4482-82f0-a46a2801c4c6 kind: Event lastTimestamp: "2026-05-20T14:14:03Z" message: InferenceGraph [sequence-graph-14a90] is Ready metadata: creationTimestamp: "2026-05-20T14:14:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:14:03Z" name: sequence-graph-14a90.18b14b97e3e45285 namespace: kserve-ci-e2e-test resourceVersion: "21202" uid: 995a2aee-57d9-406a-979b-b34d7104a2d1 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T14:15:13.823606Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-734fc-699db48bbd-8gvbt namespace: kserve-ci-e2e-test resourceVersion: "22009" uid: 881877ca-fd37-439c-b536-73be79de984c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-734fc-699db48bbd-8gvbt to ip-10-0-136-75.ec2.internal metadata: creationTimestamp: "2026-05-20T14:15:13Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T14:15:13Z" name: sequence-graph-734fc-699db48bbd-8gvbt.18b14ba84219785b namespace: kserve-ci-e2e-test resourceVersion: "22014" uid: af1bdb7c-cbce-4a27-a0c2-7b0eb58e6464 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-747dc7bb84-jx2kx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:15:14Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-734fc-699db48bbd-8gvbt namespace: kserve-ci-e2e-test resourceVersion: "22011" uid: 881877ca-fd37-439c-b536-73be79de984c kind: Event lastTimestamp: "2026-05-20T14:15:14Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-734fc-serving-cert" not found' metadata: creationTimestamp: "2026-05-20T14:15:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:15:14Z" name: sequence-graph-734fc-699db48bbd-8gvbt.18b14ba84d660c12 namespace: kserve-ci-e2e-test resourceVersion: "22029" uid: 48a09a5b-ae36-4540-ab35-d124960f8c33 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:15:14Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-734fc-699db48bbd-8gvbt namespace: kserve-ci-e2e-test resourceVersion: "22020" uid: 881877ca-fd37-439c-b536-73be79de984c kind: Event lastTimestamp: "2026-05-20T14:15:14Z" message: Add eth0 [10.134.0.45/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T14:15:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T14:15:14Z" name: sequence-graph-734fc-699db48bbd-8gvbt.18b14ba8802e0a7b namespace: kserve-ci-e2e-test resourceVersion: "22038" uid: 94922298-0994-4f3e-aaea-b8e757001557 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:15:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-734fc} kind: Pod name: sequence-graph-734fc-699db48bbd-8gvbt namespace: kserve-ci-e2e-test resourceVersion: "22011" uid: 881877ca-fd37-439c-b536-73be79de984c kind: Event lastTimestamp: "2026-05-20T14:15:14Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:04926a9d7185ff179745ae32ed77ae962904508dbcb7541e9bf600392af6f503" already present on machine metadata: creationTimestamp: "2026-05-20T14:15:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:15:14Z" name: sequence-graph-734fc-699db48bbd-8gvbt.18b14ba8817adafd namespace: kserve-ci-e2e-test resourceVersion: "22040" uid: 32212d04-a4d6-4d71-968d-0d74094a48da reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:15:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-734fc} kind: Pod name: sequence-graph-734fc-699db48bbd-8gvbt namespace: kserve-ci-e2e-test resourceVersion: "22011" uid: 881877ca-fd37-439c-b536-73be79de984c kind: Event lastTimestamp: "2026-05-20T14:15:14Z" message: 'Created container: sequence-graph-734fc' metadata: creationTimestamp: "2026-05-20T14:15:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:15:14Z" name: sequence-graph-734fc-699db48bbd-8gvbt.18b14ba884775d47 namespace: kserve-ci-e2e-test resourceVersion: "22041" uid: d91a817f-4089-4397-b4b2-8b47dc128791 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:15:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-734fc} kind: Pod name: sequence-graph-734fc-699db48bbd-8gvbt namespace: kserve-ci-e2e-test resourceVersion: "22011" uid: 881877ca-fd37-439c-b536-73be79de984c kind: Event lastTimestamp: "2026-05-20T14:15:14Z" message: Started container sequence-graph-734fc metadata: creationTimestamp: "2026-05-20T14:15:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:15:14Z" name: sequence-graph-734fc-699db48bbd-8gvbt.18b14ba884f56e6a namespace: kserve-ci-e2e-test resourceVersion: "22042" uid: 1ba2d4f7-6eae-4386-887b-5aad06e1e5c2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:23:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-734fc} kind: Pod name: sequence-graph-734fc-699db48bbd-8gvbt namespace: kserve-ci-e2e-test resourceVersion: "22011" uid: 881877ca-fd37-439c-b536-73be79de984c kind: Event lastTimestamp: "2026-05-20T14:23:28Z" message: Stopping container sequence-graph-734fc metadata: creationTimestamp: "2026-05-20T14:23:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:23:28Z" name: sequence-graph-734fc-699db48bbd-8gvbt.18b14c1b69bda91d namespace: kserve-ci-e2e-test resourceVersion: "25609" uid: 0f0cff9b-202a-4404-9d6e-1295c2e0c279 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-20T14:23:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-734fc} kind: Pod name: sequence-graph-734fc-699db48bbd-8gvbt namespace: kserve-ci-e2e-test resourceVersion: "22011" uid: 881877ca-fd37-439c-b536-73be79de984c kind: Event lastTimestamp: "2026-05-20T14:23:56Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-20T14:23:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:23:56Z" name: sequence-graph-734fc-699db48bbd-8gvbt.18b14c1c22d14efc namespace: kserve-ci-e2e-test resourceVersion: "26270" uid: b192c14e-e17f-48a9-b1da-60b4a6f8093c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:15:13Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-734fc-699db48bbd namespace: kserve-ci-e2e-test resourceVersion: "22006" uid: e484d8cb-a963-4d78-877f-147f1dbb696b kind: Event lastTimestamp: "2026-05-20T14:15:13Z" message: 'Created pod: sequence-graph-734fc-699db48bbd-8gvbt' metadata: creationTimestamp: "2026-05-20T14:15:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:15:13Z" name: sequence-graph-734fc-699db48bbd.18b14ba84194bdbd namespace: kserve-ci-e2e-test resourceVersion: "22010" uid: f150fb34-c4f0-4416-b4b8-04eb7ee484c4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:15:13Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-734fc namespace: kserve-ci-e2e-test resourceVersion: "22003" uid: 90a3ee3c-27b5-4cc7-a25c-14ffe71c711f kind: Event lastTimestamp: "2026-05-20T14:15:13Z" message: Scaled up replica set sequence-graph-734fc-699db48bbd from 0 to 1 metadata: creationTimestamp: "2026-05-20T14:15:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:15:13Z" name: sequence-graph-734fc.18b14ba8408db9a2 namespace: kserve-ci-e2e-test resourceVersion: "22007" uid: 53aba774-0bc9-4ab1-9a03-2f605dd6a0ca reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:15:13Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-734fc namespace: kserve-ci-e2e-test resourceVersion: "22023" uid: 39e7851c-2b81-4e40-b7e1-f3b5c58ec4ba kind: Event lastTimestamp: "2026-05-20T14:15:13Z" message: InferenceGraph [sequence-graph-734fc] is Ready metadata: creationTimestamp: "2026-05-20T14:15:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:15:13Z" name: sequence-graph-734fc.18b14ba844e0c9b1 namespace: kserve-ci-e2e-test resourceVersion: "22025" uid: 6c991728-a8b3-4e2c-b4ca-1df0f6760f78 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:15:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-734fc namespace: kserve-ci-e2e-test resourceVersion: "22005" uid: e61e66d3-d48b-4986-85fe-272c9e02ba8e kind: Event lastTimestamp: "2026-05-20T14:15:28Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:15:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:15:28Z" name: sequence-graph-734fc.18b14babbefa1484 namespace: kserve-ci-e2e-test resourceVersion: "22160" uid: 7eeda4ad-5e57-46f8-ac59-623825fe0dab reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:15:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-734fc namespace: kserve-ci-e2e-test resourceVersion: "22005" uid: e61e66d3-d48b-4986-85fe-272c9e02ba8e kind: Event lastTimestamp: "2026-05-20T14:15:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:15:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:15:28Z" name: sequence-graph-734fc.18b14babbf6df0e6 namespace: kserve-ci-e2e-test resourceVersion: "22162" uid: f4bfa5cf-56a9-4ced-8ace-462c7b69e215 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:15:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-734fc namespace: kserve-ci-e2e-test resourceVersion: "22161" uid: e61e66d3-d48b-4986-85fe-272c9e02ba8e kind: Event lastTimestamp: "2026-05-20T14:15:43Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:15:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:15:43Z" name: sequence-graph-734fc.18b14baf3e34b8e0 namespace: kserve-ci-e2e-test resourceVersion: "22264" uid: 5abcfee6-e927-4cd6-bc96-c129b7eb3e2c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:15:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-734fc namespace: kserve-ci-e2e-test resourceVersion: "22161" uid: e61e66d3-d48b-4986-85fe-272c9e02ba8e kind: Event lastTimestamp: "2026-05-20T14:15:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:15:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:15:43Z" name: sequence-graph-734fc.18b14baf3ebd0ad7 namespace: kserve-ci-e2e-test resourceVersion: "22266" uid: 8dd1cf63-90c3-47e1-8c41-897df26893c0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-05-20T14:04:48.596623Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-cdd1d-84988c9c55-pflbb namespace: kserve-ci-e2e-test resourceVersion: "16778" uid: f9fc80aa-7953-49f6-9d9d-3b3576cd0954 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-cdd1d-84988c9c55-pflbb to ip-10-0-136-75.ec2.internal metadata: creationTimestamp: "2026-05-20T14:04:48Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T14:04:48Z" name: sequence-graph-cdd1d-84988c9c55-pflbb.18b14b16afaae163 namespace: kserve-ci-e2e-test resourceVersion: "16783" uid: fe02ece9-f5ed-432f-a9ba-55ed7fe3d78b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-747dc7bb84-jx2kx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:04:49Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-cdd1d-84988c9c55-pflbb namespace: kserve-ci-e2e-test resourceVersion: "16787" uid: f9fc80aa-7953-49f6-9d9d-3b3576cd0954 kind: Event lastTimestamp: "2026-05-20T14:04:49Z" message: Add eth0 [10.134.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T14:04:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T14:04:49Z" name: sequence-graph-cdd1d-84988c9c55-pflbb.18b14b16c940c6b8 namespace: kserve-ci-e2e-test resourceVersion: "16805" uid: 28e6135e-10e1-45c4-b41d-7368656c8455 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:04:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-cdd1d} kind: Pod name: sequence-graph-cdd1d-84988c9c55-pflbb namespace: kserve-ci-e2e-test resourceVersion: "16780" uid: f9fc80aa-7953-49f6-9d9d-3b3576cd0954 kind: Event lastTimestamp: "2026-05-20T14:04:49Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:04926a9d7185ff179745ae32ed77ae962904508dbcb7541e9bf600392af6f503" already present on machine metadata: creationTimestamp: "2026-05-20T14:04:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:04:49Z" name: sequence-graph-cdd1d-84988c9c55-pflbb.18b14b16cab6d48d namespace: kserve-ci-e2e-test resourceVersion: "16808" uid: 94eb17a6-f8ef-435a-a690-21e4f28fd135 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:04:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-cdd1d} kind: Pod name: sequence-graph-cdd1d-84988c9c55-pflbb namespace: kserve-ci-e2e-test resourceVersion: "16780" uid: f9fc80aa-7953-49f6-9d9d-3b3576cd0954 kind: Event lastTimestamp: "2026-05-20T14:04:49Z" message: 'Created container: sequence-graph-cdd1d' metadata: creationTimestamp: "2026-05-20T14:04:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:04:49Z" name: sequence-graph-cdd1d-84988c9c55-pflbb.18b14b16ce6de98d namespace: kserve-ci-e2e-test resourceVersion: "16809" uid: 05b9ecdc-c195-47f2-bbe9-53c1cc453ad5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:04:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-cdd1d} kind: Pod name: sequence-graph-cdd1d-84988c9c55-pflbb namespace: kserve-ci-e2e-test resourceVersion: "16780" uid: f9fc80aa-7953-49f6-9d9d-3b3576cd0954 kind: Event lastTimestamp: "2026-05-20T14:04:49Z" message: Started container sequence-graph-cdd1d metadata: creationTimestamp: "2026-05-20T14:04:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:04:49Z" name: sequence-graph-cdd1d-84988c9c55-pflbb.18b14b16ced7c192 namespace: kserve-ci-e2e-test resourceVersion: "16810" uid: 97719cb4-00dd-4da2-9b0b-90b73dcb7fa1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-cdd1d} kind: Pod name: sequence-graph-cdd1d-84988c9c55-pflbb namespace: kserve-ci-e2e-test resourceVersion: "16780" uid: f9fc80aa-7953-49f6-9d9d-3b3576cd0954 kind: Event lastTimestamp: "2026-05-20T14:13:03Z" message: Stopping container sequence-graph-cdd1d metadata: creationTimestamp: "2026-05-20T14:13:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:13:03Z" name: sequence-graph-cdd1d-84988c9c55-pflbb.18b14b89db5ecf83 namespace: kserve-ci-e2e-test resourceVersion: "20356" uid: 0a85bf8e-ebe4-4c74-8c94-1f8c5339a398 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-20T14:13:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-cdd1d} kind: Pod name: sequence-graph-cdd1d-84988c9c55-pflbb namespace: kserve-ci-e2e-test resourceVersion: "16780" uid: f9fc80aa-7953-49f6-9d9d-3b3576cd0954 kind: Event lastTimestamp: "2026-05-20T14:13:30Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-20T14:13:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:13:30Z" name: sequence-graph-cdd1d-84988c9c55-pflbb.18b14b8a55099c58 namespace: kserve-ci-e2e-test resourceVersion: "20710" uid: c74c6796-79a5-44a0-b961-3c9b9baad970 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:04:48Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-cdd1d-84988c9c55 namespace: kserve-ci-e2e-test resourceVersion: "16775" uid: ccfc87e7-6193-4d7c-b562-2544610c5a80 kind: Event lastTimestamp: "2026-05-20T14:04:48Z" message: 'Created pod: sequence-graph-cdd1d-84988c9c55-pflbb' metadata: creationTimestamp: "2026-05-20T14:04:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:04:48Z" name: sequence-graph-cdd1d-84988c9c55.18b14b16aef0d509 namespace: kserve-ci-e2e-test resourceVersion: "16779" uid: 554b7ee8-7561-4b02-b944-ae341b682a4d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:04:48Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-cdd1d namespace: kserve-ci-e2e-test resourceVersion: "16773" uid: 2e074f71-fabe-4e0f-b61a-cb67856dff18 kind: Event lastTimestamp: "2026-05-20T14:04:48Z" message: Scaled up replica set sequence-graph-cdd1d-84988c9c55 from 0 to 1 metadata: creationTimestamp: "2026-05-20T14:04:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:04:48Z" name: sequence-graph-cdd1d.18b14b16ada254f8 namespace: kserve-ci-e2e-test resourceVersion: "16777" uid: 31a86b0d-2591-4db5-9bb5-4f33ea515e84 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:04:48Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-cdd1d namespace: kserve-ci-e2e-test resourceVersion: "16795" uid: c3adae53-9017-436e-9d8d-58598fa3e0c1 kind: Event lastTimestamp: "2026-05-20T14:04:48Z" message: InferenceGraph [sequence-graph-cdd1d] is Ready metadata: creationTimestamp: "2026-05-20T14:04:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:04:48Z" name: sequence-graph-cdd1d.18b14b16b61e3a5e namespace: kserve-ci-e2e-test resourceVersion: "16797" uid: b6538df4-7111-49bc-bcb5-3963b6b58c99 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:05:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-cdd1d namespace: kserve-ci-e2e-test resourceVersion: "16774" uid: a9dff175-1a91-4b58-bf2d-91d9702c4023 kind: Event lastTimestamp: "2026-05-20T14:05:03Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:05:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:05:03Z" name: sequence-graph-cdd1d.18b14b1a2c7a86aa namespace: kserve-ci-e2e-test resourceVersion: "16917" uid: 71584c31-0398-4ae1-abab-d8d3d3f05306 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:05:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-cdd1d namespace: kserve-ci-e2e-test resourceVersion: "16774" uid: a9dff175-1a91-4b58-bf2d-91d9702c4023 kind: Event lastTimestamp: "2026-05-20T14:05:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:05:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:05:03Z" name: sequence-graph-cdd1d.18b14b1a2cff356d namespace: kserve-ci-e2e-test resourceVersion: "16919" uid: 11d29dc0-7c74-4e1e-8815-9fb8085c7401 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-05-20T14:25:09.179657Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-31b2e-5cc6cdf9c9-jndff namespace: kserve-ci-e2e-test resourceVersion: "27138" uid: 029acfcc-1456-4f49-85a1-a95d8ab677d1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-31b2e-5cc6cdf9c9-jndff to ip-10-0-136-75.ec2.internal metadata: creationTimestamp: "2026-05-20T14:25:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T14:25:09Z" name: splitter-graph-31b2e-5cc6cdf9c9-jndff.18b14c32e015eb29 namespace: kserve-ci-e2e-test resourceVersion: "27143" uid: 3b86a3b0-8267-47a3-aae4-2f706a20a04f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-747dc7bb84-jx2kx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:25:09Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-31b2e-5cc6cdf9c9-jndff namespace: kserve-ci-e2e-test resourceVersion: "27141" uid: 029acfcc-1456-4f49-85a1-a95d8ab677d1 kind: Event lastTimestamp: "2026-05-20T14:25:09Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-31b2e-serving-cert" not found' metadata: creationTimestamp: "2026-05-20T14:25:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:25:09Z" name: splitter-graph-31b2e-5cc6cdf9c9-jndff.18b14c32e7e98f7d namespace: kserve-ci-e2e-test resourceVersion: "27156" uid: 10920e33-fbdb-469f-a811-87c1ed137703 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:25:10Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-31b2e-5cc6cdf9c9-jndff namespace: kserve-ci-e2e-test resourceVersion: "27146" uid: 029acfcc-1456-4f49-85a1-a95d8ab677d1 kind: Event lastTimestamp: "2026-05-20T14:25:10Z" message: Add eth0 [10.134.0.51/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T14:25:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T14:25:10Z" name: splitter-graph-31b2e-5cc6cdf9c9-jndff.18b14c331de8b253 namespace: kserve-ci-e2e-test resourceVersion: "27165" uid: 6f03a3ee-9610-41c8-a722-f556970bed12 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:25:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-31b2e} kind: Pod name: splitter-graph-31b2e-5cc6cdf9c9-jndff namespace: kserve-ci-e2e-test resourceVersion: "27141" uid: 029acfcc-1456-4f49-85a1-a95d8ab677d1 kind: Event lastTimestamp: "2026-05-20T14:25:10Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:04926a9d7185ff179745ae32ed77ae962904508dbcb7541e9bf600392af6f503" already present on machine metadata: creationTimestamp: "2026-05-20T14:25:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:25:10Z" name: splitter-graph-31b2e-5cc6cdf9c9-jndff.18b14c331f445215 namespace: kserve-ci-e2e-test resourceVersion: "27168" uid: 0064d801-4a79-4c5f-987e-25eab92d3e4d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:25:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-31b2e} kind: Pod name: splitter-graph-31b2e-5cc6cdf9c9-jndff namespace: kserve-ci-e2e-test resourceVersion: "27141" uid: 029acfcc-1456-4f49-85a1-a95d8ab677d1 kind: Event lastTimestamp: "2026-05-20T14:25:10Z" message: 'Created container: splitter-graph-31b2e' metadata: creationTimestamp: "2026-05-20T14:25:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:25:10Z" name: splitter-graph-31b2e-5cc6cdf9c9-jndff.18b14c33224b75aa namespace: kserve-ci-e2e-test resourceVersion: "27169" uid: 71c8ef82-7a0a-4241-a047-945f0eb80ae5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:25:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-31b2e} kind: Pod name: splitter-graph-31b2e-5cc6cdf9c9-jndff namespace: kserve-ci-e2e-test resourceVersion: "27141" uid: 029acfcc-1456-4f49-85a1-a95d8ab677d1 kind: Event lastTimestamp: "2026-05-20T14:25:10Z" message: Started container splitter-graph-31b2e metadata: creationTimestamp: "2026-05-20T14:25:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:25:10Z" name: splitter-graph-31b2e-5cc6cdf9c9-jndff.18b14c3322c6d62b namespace: kserve-ci-e2e-test resourceVersion: "27170" uid: aa29b9dc-6a85-4dbc-9849-1952277b50ae reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:33:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-31b2e} kind: Pod name: splitter-graph-31b2e-5cc6cdf9c9-jndff namespace: kserve-ci-e2e-test resourceVersion: "27141" uid: 029acfcc-1456-4f49-85a1-a95d8ab677d1 kind: Event lastTimestamp: "2026-05-20T14:33:23Z" message: Stopping container splitter-graph-31b2e metadata: creationTimestamp: "2026-05-20T14:33:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:33:23Z" name: splitter-graph-31b2e-5cc6cdf9c9-jndff.18b14ca60b003c58 namespace: kserve-ci-e2e-test resourceVersion: "30577" uid: 6054e8b5-bfa9-4a07-bdff-ddca3e53fd1b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-20T14:33:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-31b2e} kind: Pod name: splitter-graph-31b2e-5cc6cdf9c9-jndff namespace: kserve-ci-e2e-test resourceVersion: "27141" uid: 029acfcc-1456-4f49-85a1-a95d8ab677d1 kind: Event lastTimestamp: "2026-05-20T14:33:51Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-20T14:33:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:33:51Z" name: splitter-graph-31b2e-5cc6cdf9c9-jndff.18b14ca6b228bedb namespace: kserve-ci-e2e-test resourceVersion: "30805" uid: 4a991d04-6cf3-498c-b85c-cd935176ee81 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:25:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-31b2e-5cc6cdf9c9 namespace: kserve-ci-e2e-test resourceVersion: "27134" uid: 34b88c04-116e-4ba0-8c0a-2fb556e3b4ed kind: Event lastTimestamp: "2026-05-20T14:25:09Z" message: 'Created pod: splitter-graph-31b2e-5cc6cdf9c9-jndff' metadata: creationTimestamp: "2026-05-20T14:25:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:25:09Z" name: splitter-graph-31b2e-5cc6cdf9c9.18b14c32df86420f namespace: kserve-ci-e2e-test resourceVersion: "27140" uid: 7b9fd8b0-7008-4356-b4b8-bf2baf6afec0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:25:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-31b2e namespace: kserve-ci-e2e-test resourceVersion: "27131" uid: 11f0e62a-90b4-4116-9b9f-695b4563ef55 kind: Event lastTimestamp: "2026-05-20T14:25:09Z" message: Scaled up replica set splitter-graph-31b2e-5cc6cdf9c9 from 0 to 1 metadata: creationTimestamp: "2026-05-20T14:25:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:25:09Z" name: splitter-graph-31b2e.18b14c32de56a16b namespace: kserve-ci-e2e-test resourceVersion: "27136" uid: 6068fa0c-0270-441c-8846-fea73a91ba32 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:25:09Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-31b2e namespace: kserve-ci-e2e-test resourceVersion: "27149" uid: d4b92e33-f1b0-4134-b12c-958d0a48d900 kind: Event lastTimestamp: "2026-05-20T14:25:09Z" message: InferenceGraph [splitter-graph-31b2e] is Ready metadata: creationTimestamp: "2026-05-20T14:25:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:25:09Z" name: splitter-graph-31b2e.18b14c32e4bbba3e namespace: kserve-ci-e2e-test resourceVersion: "27151" uid: a4818b99-d851-4d96-b78f-534d10e8c317 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:25:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-31b2e namespace: kserve-ci-e2e-test resourceVersion: "27133" uid: 1d7a2cee-ba68-4572-a7d5-3eed8d9de477 kind: Event lastTimestamp: "2026-05-20T14:25:24Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:25:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:25:24Z" name: splitter-graph-31b2e.18b14c365d3812bd namespace: kserve-ci-e2e-test resourceVersion: "27274" uid: 91c98ffa-893a-4bb8-b55b-d6f116c8a90c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:25:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-31b2e namespace: kserve-ci-e2e-test resourceVersion: "27133" uid: 1d7a2cee-ba68-4572-a7d5-3eed8d9de477 kind: Event lastTimestamp: "2026-05-20T14:25:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:25:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:25:24Z" name: splitter-graph-31b2e.18b14c365da7ef5f namespace: kserve-ci-e2e-test resourceVersion: "27275" uid: 1f06d65b-4e8e-4b08-acd6-dd2a590a03db reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:25:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-31b2e namespace: kserve-ci-e2e-test resourceVersion: "27273" uid: 1d7a2cee-ba68-4572-a7d5-3eed8d9de477 kind: Event lastTimestamp: "2026-05-20T14:25:39Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:25:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:25:39Z" name: splitter-graph-31b2e.18b14c39dc611d4a namespace: kserve-ci-e2e-test resourceVersion: "27378" uid: 16dde687-2b05-42f3-bbf0-48fbaa1e0afd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:25:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-31b2e namespace: kserve-ci-e2e-test resourceVersion: "27273" uid: 1d7a2cee-ba68-4572-a7d5-3eed8d9de477 kind: Event lastTimestamp: "2026-05-20T14:25:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:25:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:25:39Z" name: splitter-graph-31b2e.18b14c39dd02955b namespace: kserve-ci-e2e-test resourceVersion: "27380" uid: 04392c08-2838-4b1d-940b-8e76f6d78a35 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-05-20T14:23:58.861428Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-a868a-757c8c8c4d-92495 namespace: kserve-ci-e2e-test resourceVersion: "26307" uid: ea4f12b1-b727-4900-b2ea-5a7afc7bd513 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-a868a-757c8c8c4d-92495 to ip-10-0-136-75.ec2.internal metadata: creationTimestamp: "2026-05-20T14:23:58Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T14:23:58Z" name: splitter-graph-a868a-757c8c8c4d-92495.18b14c2280cae545 namespace: kserve-ci-e2e-test resourceVersion: "26312" uid: 2a893df6-c42e-4a77-8579-882925fdc88a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-747dc7bb84-jx2kx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:23:59Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-a868a-757c8c8c4d-92495 namespace: kserve-ci-e2e-test resourceVersion: "26315" uid: ea4f12b1-b727-4900-b2ea-5a7afc7bd513 kind: Event lastTimestamp: "2026-05-20T14:23:59Z" message: Add eth0 [10.134.0.48/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T14:23:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T14:23:59Z" name: splitter-graph-a868a-757c8c8c4d-92495.18b14c229b4f984a namespace: kserve-ci-e2e-test resourceVersion: "26331" uid: c94228d3-d892-4d58-af82-ff3d20fecf0c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:23:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-a868a} kind: Pod name: splitter-graph-a868a-757c8c8c4d-92495 namespace: kserve-ci-e2e-test resourceVersion: "26309" uid: ea4f12b1-b727-4900-b2ea-5a7afc7bd513 kind: Event lastTimestamp: "2026-05-20T14:23:59Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:04926a9d7185ff179745ae32ed77ae962904508dbcb7541e9bf600392af6f503" already present on machine metadata: creationTimestamp: "2026-05-20T14:23:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:23:59Z" name: splitter-graph-a868a-757c8c8c4d-92495.18b14c229ca65ad2 namespace: kserve-ci-e2e-test resourceVersion: "26333" uid: f7a08edf-827d-4bf6-9750-ecfb86055b42 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:23:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-a868a} kind: Pod name: splitter-graph-a868a-757c8c8c4d-92495 namespace: kserve-ci-e2e-test resourceVersion: "26309" uid: ea4f12b1-b727-4900-b2ea-5a7afc7bd513 kind: Event lastTimestamp: "2026-05-20T14:23:59Z" message: 'Created container: splitter-graph-a868a' metadata: creationTimestamp: "2026-05-20T14:23:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:23:59Z" name: splitter-graph-a868a-757c8c8c4d-92495.18b14c22a03970e1 namespace: kserve-ci-e2e-test resourceVersion: "26336" uid: 9a3738e0-8eb5-4917-b6e9-41730b62bfa3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:23:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-a868a} kind: Pod name: splitter-graph-a868a-757c8c8c4d-92495 namespace: kserve-ci-e2e-test resourceVersion: "26309" uid: ea4f12b1-b727-4900-b2ea-5a7afc7bd513 kind: Event lastTimestamp: "2026-05-20T14:23:59Z" message: Started container splitter-graph-a868a metadata: creationTimestamp: "2026-05-20T14:23:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:23:59Z" name: splitter-graph-a868a-757c8c8c4d-92495.18b14c22a0b48955 namespace: kserve-ci-e2e-test resourceVersion: "26337" uid: 724bb0a7-dca2-4893-ae0d-652cd6659f62 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:24:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-a868a} kind: Pod name: splitter-graph-a868a-757c8c8c4d-92495 namespace: kserve-ci-e2e-test resourceVersion: "26309" uid: ea4f12b1-b727-4900-b2ea-5a7afc7bd513 kind: Event lastTimestamp: "2026-05-20T14:24:08Z" message: Stopping container splitter-graph-a868a metadata: creationTimestamp: "2026-05-20T14:24:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:24:08Z" name: splitter-graph-a868a-757c8c8c4d-92495.18b14c24daf64cd2 namespace: kserve-ci-e2e-test resourceVersion: "26436" uid: 9c425b96-68f2-44e7-9046-9120d5eebfd0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-20T14:24:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-a868a} kind: Pod name: splitter-graph-a868a-757c8c8c4d-92495 namespace: kserve-ci-e2e-test resourceVersion: "26309" uid: ea4f12b1-b727-4900-b2ea-5a7afc7bd513 kind: Event lastTimestamp: "2026-05-20T14:24:36Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-20T14:24:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:24:36Z" name: splitter-graph-a868a-757c8c8c4d-92495.18b14c2569b524c8 namespace: kserve-ci-e2e-test resourceVersion: "26843" uid: 55f644ed-5e64-46a3-b417-220e49bd5193 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:23:58Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-a868a-757c8c8c4d namespace: kserve-ci-e2e-test resourceVersion: "26304" uid: df418906-e14c-46d5-833c-4ab1471782ed kind: Event lastTimestamp: "2026-05-20T14:23:58Z" message: 'Created pod: splitter-graph-a868a-757c8c8c4d-92495' metadata: creationTimestamp: "2026-05-20T14:23:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:23:58Z" name: splitter-graph-a868a-757c8c8c4d.18b14c2280595b5e namespace: kserve-ci-e2e-test resourceVersion: "26308" uid: b147f691-d10d-4ec8-a0cd-66be9025d19a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:23:58Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-a868a namespace: kserve-ci-e2e-test resourceVersion: "26301" uid: 1acd29b4-3787-481d-80a9-a1f568bfdf29 kind: Event lastTimestamp: "2026-05-20T14:23:58Z" message: Scaled up replica set splitter-graph-a868a-757c8c8c4d from 0 to 1 metadata: creationTimestamp: "2026-05-20T14:23:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:23:58Z" name: splitter-graph-a868a.18b14c227f31e769 namespace: kserve-ci-e2e-test resourceVersion: "26305" uid: 81004ce6-735b-4151-ae96-f5eac9e54ecb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:23:58Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-a868a namespace: kserve-ci-e2e-test resourceVersion: "26320" uid: 241cea49-e28a-4679-b259-b55b9b6be451 kind: Event lastTimestamp: "2026-05-20T14:23:58Z" message: InferenceGraph [splitter-graph-a868a] is Ready metadata: creationTimestamp: "2026-05-20T14:23:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:23:58Z" name: splitter-graph-a868a.18b14c228532b447 namespace: kserve-ci-e2e-test resourceVersion: "26323" uid: bab8c4f4-edfd-4cef-a397-cc01824ea5af reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T14:13:03.450400Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-14a90-predictor-75bc8f9bb-fv57c namespace: kserve-ci-e2e-test resourceVersion: "20387" uid: 78117ed5-88b7-4c96-8676-0f39b68f132d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-14a90-predictor-75bc8f9bb-fv57c to ip-10-0-137-148.ec2.internal metadata: creationTimestamp: "2026-05-20T14:13:03Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T14:13:03Z" name: success-200-isvc-14a90-predictor-75bc8f9bb-fv57c.18b14b89e7403f6a namespace: kserve-ci-e2e-test resourceVersion: "20395" uid: 3cc908bc-c279-4f70-a17a-7bc902edd61c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-747dc7bb84-jx2kx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:03Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-14a90-predictor-75bc8f9bb-fv57c namespace: kserve-ci-e2e-test resourceVersion: "20404" uid: 78117ed5-88b7-4c96-8676-0f39b68f132d kind: Event lastTimestamp: "2026-05-20T14:13:03Z" message: Add eth0 [10.133.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T14:13:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T14:13:03Z" name: success-200-isvc-14a90-predictor-75bc8f9bb-fv57c.18b14b8a00b4802c namespace: kserve-ci-e2e-test resourceVersion: "20452" uid: faadf62a-5637-4a89-ad3f-c72eb94727bd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-14a90-predictor-75bc8f9bb-fv57c namespace: kserve-ci-e2e-test resourceVersion: "20392" uid: 78117ed5-88b7-4c96-8676-0f39b68f132d kind: Event lastTimestamp: "2026-05-20T14:13:03Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1501" already present on machine metadata: creationTimestamp: "2026-05-20T14:13:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:13:03Z" name: success-200-isvc-14a90-predictor-75bc8f9bb-fv57c.18b14b8a029488fe namespace: kserve-ci-e2e-test resourceVersion: "20455" uid: 12680032-c309-451d-9424-b90c29ff7e36 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-14a90-predictor-75bc8f9bb-fv57c namespace: kserve-ci-e2e-test resourceVersion: "20392" uid: 78117ed5-88b7-4c96-8676-0f39b68f132d kind: Event lastTimestamp: "2026-05-20T14:13:03Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T14:13:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:13:03Z" name: success-200-isvc-14a90-predictor-75bc8f9bb-fv57c.18b14b8a059be3df namespace: kserve-ci-e2e-test resourceVersion: "20459" uid: 4024adfb-8eed-4985-99d2-456ba427b5c4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-14a90-predictor-75bc8f9bb-fv57c namespace: kserve-ci-e2e-test resourceVersion: "20392" uid: 78117ed5-88b7-4c96-8676-0f39b68f132d kind: Event lastTimestamp: "2026-05-20T14:13:03Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T14:13:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:13:03Z" name: success-200-isvc-14a90-predictor-75bc8f9bb-fv57c.18b14b8a060ec63f namespace: kserve-ci-e2e-test resourceVersion: "20460" uid: 1670afb3-4374-4ed9-840d-9040f182d2fc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-14a90-predictor-75bc8f9bb-fv57c namespace: kserve-ci-e2e-test resourceVersion: "20392" uid: 78117ed5-88b7-4c96-8676-0f39b68f132d kind: Event lastTimestamp: "2026-05-20T14:13:03Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-20T14:13:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:13:03Z" name: success-200-isvc-14a90-predictor-75bc8f9bb-fv57c.18b14b8a061a7914 namespace: kserve-ci-e2e-test resourceVersion: "20461" uid: 8b5c4deb-7e12-47ce-9886-edd0d9a4dc28 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-14a90-predictor-75bc8f9bb-fv57c namespace: kserve-ci-e2e-test resourceVersion: "20392" uid: 78117ed5-88b7-4c96-8676-0f39b68f132d kind: Event lastTimestamp: "2026-05-20T14:13:04Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T14:13:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:13:04Z" name: success-200-isvc-14a90-predictor-75bc8f9bb-fv57c.18b14b8a08da1519 namespace: kserve-ci-e2e-test resourceVersion: "20462" uid: a197ff85-5468-4d9e-8e7c-2367a1dcf64f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-14a90-predictor-75bc8f9bb-fv57c namespace: kserve-ci-e2e-test resourceVersion: "20392" uid: 78117ed5-88b7-4c96-8676-0f39b68f132d kind: Event lastTimestamp: "2026-05-20T14:13:04Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:13:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:13:04Z" name: success-200-isvc-14a90-predictor-75bc8f9bb-fv57c.18b14b8a0949c9dd namespace: kserve-ci-e2e-test resourceVersion: "20463" uid: a0a86fca-b36d-4bd2-ae80-a97d1a99c803 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-20T14:13:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-14a90-predictor-75bc8f9bb-fv57c namespace: kserve-ci-e2e-test resourceVersion: "20392" uid: 78117ed5-88b7-4c96-8676-0f39b68f132d kind: Event lastTimestamp: "2026-05-20T14:13:41Z" message: 'Readiness probe failed: dial tcp 10.133.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T14:13:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:13:42Z" name: success-200-isvc-14a90-predictor-75bc8f9bb-fv57c.18b14b8a7e914702 namespace: kserve-ci-e2e-test resourceVersion: "20845" uid: 201c1cd2-d9a4-4921-90f8-8542abb52098 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:14:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-14a90-predictor-75bc8f9bb-fv57c namespace: kserve-ci-e2e-test resourceVersion: "20392" uid: 78117ed5-88b7-4c96-8676-0f39b68f132d kind: Event lastTimestamp: "2026-05-20T14:14:13Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T14:14:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:14:13Z" name: success-200-isvc-14a90-predictor-75bc8f9bb-fv57c.18b14b9a43117df4 namespace: kserve-ci-e2e-test resourceVersion: "21374" uid: 9cd49215-22c5-40e6-8627-770a5d9799ca reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:14:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-14a90-predictor-75bc8f9bb-fv57c namespace: kserve-ci-e2e-test resourceVersion: "20392" uid: 78117ed5-88b7-4c96-8676-0f39b68f132d kind: Event lastTimestamp: "2026-05-20T14:14:13Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:14:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:14:13Z" name: success-200-isvc-14a90-predictor-75bc8f9bb-fv57c.18b14b9a4312abc7 namespace: kserve-ci-e2e-test resourceVersion: "21375" uid: efebe3b8-fbef-4667-b352-28497db8ad29 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:03Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-14a90-predictor-75bc8f9bb namespace: kserve-ci-e2e-test resourceVersion: "20376" uid: 9aaf8302-8b27-4b4e-99f3-d2253a6efb33 kind: Event lastTimestamp: "2026-05-20T14:13:03Z" message: 'Created pod: success-200-isvc-14a90-predictor-75bc8f9bb-fv57c' metadata: creationTimestamp: "2026-05-20T14:13:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:13:03Z" name: success-200-isvc-14a90-predictor-75bc8f9bb.18b14b89e676bf59 namespace: kserve-ci-e2e-test resourceVersion: "20393" uid: 86faa1ac-974a-4632-8681-b63fd87f0504 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:03Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-14a90-predictor namespace: kserve-ci-e2e-test resourceVersion: "20370" uid: 8f897355-4c3a-411e-9808-b95c09ffde7d kind: Event lastTimestamp: "2026-05-20T14:13:03Z" message: Scaled up replica set success-200-isvc-14a90-predictor-75bc8f9bb from 0 to 1 metadata: creationTimestamp: "2026-05-20T14:13:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:13:03Z" name: success-200-isvc-14a90-predictor.18b14b89e4005fe2 namespace: kserve-ci-e2e-test resourceVersion: "20382" uid: 5d4b65b0-5f03-4347-bff5-67015a166e6b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T14:13:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-14a90-predictor namespace: kserve-ci-e2e-test resourceVersion: "20373" uid: 70e3a057-a9eb-4635-9343-c9c649eef0c2 kind: Event lastTimestamp: "2026-05-20T14:13:33Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:13:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:13:33Z" name: success-200-isvc-14a90-predictor.18b14b8d62787d26 namespace: kserve-ci-e2e-test resourceVersion: "20732" uid: e59fb5ea-5c02-4fe7-8961-05114979459d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T14:13:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-14a90-predictor namespace: kserve-ci-e2e-test resourceVersion: "20373" uid: 70e3a057-a9eb-4635-9343-c9c649eef0c2 kind: Event lastTimestamp: "2026-05-20T14:13:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:13:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:13:33Z" name: success-200-isvc-14a90-predictor.18b14b8d62ec06f3 namespace: kserve-ci-e2e-test resourceVersion: "20733" uid: 43e0b08b-56ec-4a85-9492-52aafaab2f0a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T14:13:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-14a90-predictor namespace: kserve-ci-e2e-test resourceVersion: "20601" uid: 70e3a057-a9eb-4635-9343-c9c649eef0c2 kind: Event lastTimestamp: "2026-05-20T14:14:03Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:13:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:14:03Z" name: success-200-isvc-14a90-predictor.18b14b9460f4e0b7 namespace: kserve-ci-e2e-test resourceVersion: "21196" uid: 45ef792c-106b-4c23-b30b-0c3726658f8b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T14:13:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-14a90-predictor namespace: kserve-ci-e2e-test resourceVersion: "20601" uid: 70e3a057-a9eb-4635-9343-c9c649eef0c2 kind: Event lastTimestamp: "2026-05-20T14:14:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:13:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:14:03Z" name: success-200-isvc-14a90-predictor.18b14b9461657a6c namespace: kserve-ci-e2e-test resourceVersion: "21197" uid: a5403b1e-9fe0-4fea-91b2-86ba4c621a03 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-20T14:13:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-14a90 namespace: kserve-ci-e2e-test resourceVersion: "20358" uid: 5be0c9a3-1082-4f74-9a32-b0a4ecab8c22 kind: Event lastTimestamp: "2026-05-20T14:13:52Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T14:13:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:13:52Z" name: success-200-isvc-14a90.18b14b89e31f13eb namespace: kserve-ci-e2e-test resourceVersion: "21089" uid: 32d9f977-bda5-4672-aba9-971c39c35cad reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-14a90 namespace: kserve-ci-e2e-test resourceVersion: "20358" uid: 5be0c9a3-1082-4f74-9a32-b0a4ecab8c22 kind: Event lastTimestamp: "2026-05-20T14:13:03Z" message: 'Failed to update status for InferenceService "success-200-isvc-14a90": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-14a90": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T14:13:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:13:03Z" name: success-200-isvc-14a90.18b14b89e451aa6c namespace: kserve-ci-e2e-test resourceVersion: "20383" uid: ba32b0e3-8cea-4ddf-ad1a-48d8b207cb2e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-14a90 namespace: kserve-ci-e2e-test resourceVersion: "20358" uid: 5be0c9a3-1082-4f74-9a32-b0a4ecab8c22 kind: Event lastTimestamp: "2026-05-20T14:13:03Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-14a90": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T14:13:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:13:03Z" name: success-200-isvc-14a90.18b14b89e4521800 namespace: kserve-ci-e2e-test resourceVersion: "20385" uid: bfdcedef-7e9c-4ad1-9c42-9286f7840dbf reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-14a90 namespace: kserve-ci-e2e-test resourceVersion: "21083" uid: 5be0c9a3-1082-4f74-9a32-b0a4ecab8c22 kind: Event lastTimestamp: "2026-05-20T14:13:52Z" message: InferenceService [success-200-isvc-14a90] is Ready metadata: creationTimestamp: "2026-05-20T14:13:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:13:52Z" name: success-200-isvc-14a90.18b14b953a2437c1 namespace: kserve-ci-e2e-test resourceVersion: "21084" uid: cfae3499-7c25-40ce-8d11-3aae96cfeb22 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T14:12:33.851684Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-15cdb-predictor-679cfbb586-s4rql namespace: kserve-ci-e2e-test resourceVersion: "20049" uid: bcc9b081-db86-4142-ac1e-c60c7de5d7cf kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-15cdb-predictor-679cfbb586-s4rql to ip-10-0-137-148.ec2.internal metadata: creationTimestamp: "2026-05-20T14:12:33Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T14:12:33Z" name: success-200-isvc-15cdb-predictor-679cfbb586-s4rql.18b14b830307af37 namespace: kserve-ci-e2e-test resourceVersion: "20062" uid: ca535613-0098-44c3-8084-19cbd4597511 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-747dc7bb84-jx2kx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:12:34Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-15cdb-predictor-679cfbb586-s4rql namespace: kserve-ci-e2e-test resourceVersion: "20071" uid: bcc9b081-db86-4142-ac1e-c60c7de5d7cf kind: Event lastTimestamp: "2026-05-20T14:12:34Z" message: Add eth0 [10.133.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T14:12:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T14:12:34Z" name: success-200-isvc-15cdb-predictor-679cfbb586-s4rql.18b14b831c170a3d namespace: kserve-ci-e2e-test resourceVersion: "20089" uid: 08b58e8d-752c-4b5c-8649-1f94cc1ef2de reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:12:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-15cdb-predictor-679cfbb586-s4rql namespace: kserve-ci-e2e-test resourceVersion: "20056" uid: bcc9b081-db86-4142-ac1e-c60c7de5d7cf kind: Event lastTimestamp: "2026-05-20T14:12:34Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1501" already present on machine metadata: creationTimestamp: "2026-05-20T14:12:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:12:34Z" name: success-200-isvc-15cdb-predictor-679cfbb586-s4rql.18b14b831dfd1e53 namespace: kserve-ci-e2e-test resourceVersion: "20091" uid: 44335555-9913-4385-9705-3038a4d48068 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:12:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-15cdb-predictor-679cfbb586-s4rql namespace: kserve-ci-e2e-test resourceVersion: "20056" uid: bcc9b081-db86-4142-ac1e-c60c7de5d7cf kind: Event lastTimestamp: "2026-05-20T14:12:34Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T14:12:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:12:34Z" name: success-200-isvc-15cdb-predictor-679cfbb586-s4rql.18b14b8320a8579b namespace: kserve-ci-e2e-test resourceVersion: "20093" uid: 46b3ac6b-043c-415d-916a-a5a51deb9c35 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:12:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-15cdb-predictor-679cfbb586-s4rql namespace: kserve-ci-e2e-test resourceVersion: "20056" uid: bcc9b081-db86-4142-ac1e-c60c7de5d7cf kind: Event lastTimestamp: "2026-05-20T14:12:34Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T14:12:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:12:34Z" name: success-200-isvc-15cdb-predictor-679cfbb586-s4rql.18b14b832116c8f6 namespace: kserve-ci-e2e-test resourceVersion: "20094" uid: ea845157-02b1-4d8a-b8f2-011b55312b3f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:12:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-15cdb-predictor-679cfbb586-s4rql namespace: kserve-ci-e2e-test resourceVersion: "20056" uid: bcc9b081-db86-4142-ac1e-c60c7de5d7cf kind: Event lastTimestamp: "2026-05-20T14:12:34Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-20T14:12:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:12:34Z" name: success-200-isvc-15cdb-predictor-679cfbb586-s4rql.18b14b832122638d namespace: kserve-ci-e2e-test resourceVersion: "20096" uid: 60897678-73cd-418d-b8d5-bf49edf47758 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:12:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-15cdb-predictor-679cfbb586-s4rql namespace: kserve-ci-e2e-test resourceVersion: "20056" uid: bcc9b081-db86-4142-ac1e-c60c7de5d7cf kind: Event lastTimestamp: "2026-05-20T14:12:34Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T14:12:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:12:34Z" name: success-200-isvc-15cdb-predictor-679cfbb586-s4rql.18b14b8323cdd66b namespace: kserve-ci-e2e-test resourceVersion: "20097" uid: d957615a-e161-4a50-bed3-04323fdae33d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:12:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-15cdb-predictor-679cfbb586-s4rql namespace: kserve-ci-e2e-test resourceVersion: "20056" uid: bcc9b081-db86-4142-ac1e-c60c7de5d7cf kind: Event lastTimestamp: "2026-05-20T14:12:34Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:12:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:12:34Z" name: success-200-isvc-15cdb-predictor-679cfbb586-s4rql.18b14b832450670c namespace: kserve-ci-e2e-test resourceVersion: "20099" uid: 5027a6b3-a641-4479-9872-8d5e17421037 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-20T14:12:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-15cdb-predictor-679cfbb586-s4rql namespace: kserve-ci-e2e-test resourceVersion: "20056" uid: bcc9b081-db86-4142-ac1e-c60c7de5d7cf kind: Event lastTimestamp: "2026-05-20T14:13:10Z" message: 'Readiness probe failed: dial tcp 10.133.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T14:12:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:13:10Z" name: success-200-isvc-15cdb-predictor-679cfbb586-s4rql.18b14b83410b9bc0 namespace: kserve-ci-e2e-test resourceVersion: "20534" uid: 57a3869a-7117-43f4-ba22-9aed5872844f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-15cdb-predictor-679cfbb586-s4rql namespace: kserve-ci-e2e-test resourceVersion: "20056" uid: bcc9b081-db86-4142-ac1e-c60c7de5d7cf kind: Event lastTimestamp: "2026-05-20T14:13:43Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T14:13:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:13:43Z" name: success-200-isvc-15cdb-predictor-679cfbb586-s4rql.18b14b93525924c3 namespace: kserve-ci-e2e-test resourceVersion: "20911" uid: 5483c655-8764-4f48-9753-18bd37e27b45 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-15cdb-predictor-679cfbb586-s4rql namespace: kserve-ci-e2e-test resourceVersion: "20056" uid: bcc9b081-db86-4142-ac1e-c60c7de5d7cf kind: Event lastTimestamp: "2026-05-20T14:13:43Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:13:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:13:43Z" name: success-200-isvc-15cdb-predictor-679cfbb586-s4rql.18b14b935259d8ba namespace: kserve-ci-e2e-test resourceVersion: "20913" uid: 56de020d-ed20-4dc5-b9fb-ed9b513b64f2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-15cdb-predictor-679cfbb586-s4rql namespace: kserve-ci-e2e-test resourceVersion: "20056" uid: bcc9b081-db86-4142-ac1e-c60c7de5d7cf kind: Event lastTimestamp: "2026-05-20T14:13:45Z" message: 'Readiness probe failed: Get "https://10.133.0.24:8643/healthz": dial tcp 10.133.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-20T14:13:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:13:45Z" name: success-200-isvc-15cdb-predictor-679cfbb586-s4rql.18b14b93c91a6fca namespace: kserve-ci-e2e-test resourceVersion: "21011" uid: d557c7a0-27b2-403b-b9ae-8c4795c94c35 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:12:33Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-15cdb-predictor-679cfbb586 namespace: kserve-ci-e2e-test resourceVersion: "20019" uid: 4107ba28-2e9a-468f-ad43-3a1c6464abec kind: Event lastTimestamp: "2026-05-20T14:12:33Z" message: 'Created pod: success-200-isvc-15cdb-predictor-679cfbb586-s4rql' metadata: creationTimestamp: "2026-05-20T14:12:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:12:33Z" name: success-200-isvc-15cdb-predictor-679cfbb586.18b14b830148626c namespace: kserve-ci-e2e-test resourceVersion: "20051" uid: ab7383bc-5361-458a-a78e-7afea83fbc84 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:12:33Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-15cdb-predictor namespace: kserve-ci-e2e-test resourceVersion: "20011" uid: f714cd01-4200-4b11-8759-fbff71af494d kind: Event lastTimestamp: "2026-05-20T14:12:33Z" message: Scaled up replica set success-200-isvc-15cdb-predictor-679cfbb586 from 0 to 1 metadata: creationTimestamp: "2026-05-20T14:12:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:12:33Z" name: success-200-isvc-15cdb-predictor.18b14b82f61f7ef6 namespace: kserve-ci-e2e-test resourceVersion: "20023" uid: 3c73d6b6-375a-4be3-b953-cbebe6a897bf reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-20T14:12:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-15cdb-predictor namespace: kserve-ci-e2e-test resourceVersion: "20017" uid: ee28e1a4-270a-418a-9bac-ae0ffc399572 kind: Event lastTimestamp: "2026-05-20T14:13:33Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:12:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:13:33Z" name: success-200-isvc-15cdb-predictor.18b14b867592e316 namespace: kserve-ci-e2e-test resourceVersion: "20761" uid: 4e9a92d0-253a-4f4d-b8f5-af0153f73f47 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-20T14:12:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-15cdb-predictor namespace: kserve-ci-e2e-test resourceVersion: "20017" uid: ee28e1a4-270a-418a-9bac-ae0ffc399572 kind: Event lastTimestamp: "2026-05-20T14:13:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:12:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:13:33Z" name: success-200-isvc-15cdb-predictor.18b14b8676533061 namespace: kserve-ci-e2e-test resourceVersion: "20762" uid: 6b952ecc-90d4-48b7-a9e5-a80775f8a122 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-20T14:12:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-15cdb namespace: kserve-ci-e2e-test resourceVersion: "19996" uid: 59bc43d8-a566-4f42-84fb-efd44c46136d kind: Event lastTimestamp: "2026-05-20T14:13:21Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T14:12:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:13:21Z" name: success-200-isvc-15cdb.18b14b82f5b6f003 namespace: kserve-ci-e2e-test resourceVersion: "20647" uid: 53a2255f-e9d4-40dc-872d-208b8fcedc03 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:12:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-15cdb namespace: kserve-ci-e2e-test resourceVersion: "19996" uid: 59bc43d8-a566-4f42-84fb-efd44c46136d kind: Event lastTimestamp: "2026-05-20T14:12:33Z" message: 'Failed to update status for InferenceService "success-200-isvc-15cdb": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-15cdb": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T14:12:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:12:33Z" name: success-200-isvc-15cdb.18b14b82f686dd76 namespace: kserve-ci-e2e-test resourceVersion: "20021" uid: fdc40107-816e-454a-9ef1-491f25b0f085 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:12:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-15cdb namespace: kserve-ci-e2e-test resourceVersion: "19996" uid: 59bc43d8-a566-4f42-84fb-efd44c46136d kind: Event lastTimestamp: "2026-05-20T14:12:33Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-15cdb": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T14:12:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:12:33Z" name: success-200-isvc-15cdb.18b14b82f68737cb namespace: kserve-ci-e2e-test resourceVersion: "20025" uid: 9235ddc5-e0b6-4fc7-86ab-bf2e949f5d23 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-15cdb namespace: kserve-ci-e2e-test resourceVersion: "20641" uid: 59bc43d8-a566-4f42-84fb-efd44c46136d kind: Event lastTimestamp: "2026-05-20T14:13:21Z" message: InferenceService [success-200-isvc-15cdb] is Ready metadata: creationTimestamp: "2026-05-20T14:13:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:13:21Z" name: success-200-isvc-15cdb.18b14b8dfe06029e namespace: kserve-ci-e2e-test resourceVersion: "20642" uid: 9e50b01e-afb5-42f7-a640-52b77762543d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T14:24:09.102244Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-31b2e-predictor-67cf4677b6-p2vdx namespace: kserve-ci-e2e-test resourceVersion: "26472" uid: 1a64186b-ecb2-4088-9b56-24830d81d178 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-31b2e-predictor-67cf4677b6-p2vdx to ip-10-0-137-148.ec2.internal metadata: creationTimestamp: "2026-05-20T14:24:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T14:24:09Z" name: success-200-isvc-31b2e-predictor-67cf4677b6-p2vdx.18b14c24e33152a2 namespace: kserve-ci-e2e-test resourceVersion: "26477" uid: f1e82e7e-500e-4cc4-878e-6413e5887059 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-747dc7bb84-jx2kx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:24:09Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-31b2e-predictor-67cf4677b6-p2vdx namespace: kserve-ci-e2e-test resourceVersion: "26475" uid: 1a64186b-ecb2-4088-9b56-24830d81d178 kind: Event lastTimestamp: "2026-05-20T14:24:09Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-31b2e-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-20T14:24:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:24:09Z" name: success-200-isvc-31b2e-predictor-67cf4677b6-p2vdx.18b14c24efda4e15 namespace: kserve-ci-e2e-test resourceVersion: "26528" uid: 8802c48b-465c-4056-83c7-52f4d84167b5 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:24:10Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-31b2e-predictor-67cf4677b6-p2vdx namespace: kserve-ci-e2e-test resourceVersion: "26482" uid: 1a64186b-ecb2-4088-9b56-24830d81d178 kind: Event lastTimestamp: "2026-05-20T14:24:10Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T14:24:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T14:24:10Z" name: success-200-isvc-31b2e-predictor-67cf4677b6-p2vdx.18b14c25200a6912 namespace: kserve-ci-e2e-test resourceVersion: "26545" uid: 8aca4da8-cb1b-49dc-ab24-da69c49adf65 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:24:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-31b2e-predictor-67cf4677b6-p2vdx namespace: kserve-ci-e2e-test resourceVersion: "26475" uid: 1a64186b-ecb2-4088-9b56-24830d81d178 kind: Event lastTimestamp: "2026-05-20T14:24:10Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1501" already present on machine metadata: creationTimestamp: "2026-05-20T14:24:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:24:10Z" name: success-200-isvc-31b2e-predictor-67cf4677b6-p2vdx.18b14c25226643ef namespace: kserve-ci-e2e-test resourceVersion: "26547" uid: d5f1de57-ce75-4640-915b-72a9d3619bb6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:24:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-31b2e-predictor-67cf4677b6-p2vdx namespace: kserve-ci-e2e-test resourceVersion: "26475" uid: 1a64186b-ecb2-4088-9b56-24830d81d178 kind: Event lastTimestamp: "2026-05-20T14:24:10Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T14:24:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:24:10Z" name: success-200-isvc-31b2e-predictor-67cf4677b6-p2vdx.18b14c2525442c12 namespace: kserve-ci-e2e-test resourceVersion: "26548" uid: 76318afb-45bd-48b4-ac8c-d8db3d7e4f77 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:24:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-31b2e-predictor-67cf4677b6-p2vdx namespace: kserve-ci-e2e-test resourceVersion: "26475" uid: 1a64186b-ecb2-4088-9b56-24830d81d178 kind: Event lastTimestamp: "2026-05-20T14:24:10Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T14:24:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:24:10Z" name: success-200-isvc-31b2e-predictor-67cf4677b6-p2vdx.18b14c2525b17294 namespace: kserve-ci-e2e-test resourceVersion: "26549" uid: 1e0d5b44-23eb-4582-a16f-45c2f7c8686b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:24:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-31b2e-predictor-67cf4677b6-p2vdx namespace: kserve-ci-e2e-test resourceVersion: "26475" uid: 1a64186b-ecb2-4088-9b56-24830d81d178 kind: Event lastTimestamp: "2026-05-20T14:24:10Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-20T14:24:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:24:10Z" name: success-200-isvc-31b2e-predictor-67cf4677b6-p2vdx.18b14c2525bc9f18 namespace: kserve-ci-e2e-test resourceVersion: "26550" uid: 48d622c5-c00f-42da-b85a-87937ee56392 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:24:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-31b2e-predictor-67cf4677b6-p2vdx namespace: kserve-ci-e2e-test resourceVersion: "26475" uid: 1a64186b-ecb2-4088-9b56-24830d81d178 kind: Event lastTimestamp: "2026-05-20T14:24:10Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T14:24:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:24:10Z" name: success-200-isvc-31b2e-predictor-67cf4677b6-p2vdx.18b14c2529367383 namespace: kserve-ci-e2e-test resourceVersion: "26554" uid: a9b44c7b-30f1-4981-b3e8-ec1e08d4d6b2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:24:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-31b2e-predictor-67cf4677b6-p2vdx namespace: kserve-ci-e2e-test resourceVersion: "26475" uid: 1a64186b-ecb2-4088-9b56-24830d81d178 kind: Event lastTimestamp: "2026-05-20T14:24:10Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:24:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:24:10Z" name: success-200-isvc-31b2e-predictor-67cf4677b6-p2vdx.18b14c2529a590f2 namespace: kserve-ci-e2e-test resourceVersion: "26555" uid: 6fde184e-fa4c-4509-9edb-6289878bdd67 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-20T14:24:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-31b2e-predictor-67cf4677b6-p2vdx namespace: kserve-ci-e2e-test resourceVersion: "26475" uid: 1a64186b-ecb2-4088-9b56-24830d81d178 kind: Event lastTimestamp: "2026-05-20T14:24:47Z" message: 'Readiness probe failed: dial tcp 10.133.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T14:24:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:24:47Z" name: success-200-isvc-31b2e-predictor-67cf4677b6-p2vdx.18b14c2587f2dd18 namespace: kserve-ci-e2e-test resourceVersion: "26942" uid: 0755d8cd-6a14-4ee8-a863-2f5f71a02477 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:33:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-31b2e-predictor-67cf4677b6-p2vdx namespace: kserve-ci-e2e-test resourceVersion: "26475" uid: 1a64186b-ecb2-4088-9b56-24830d81d178 kind: Event lastTimestamp: "2026-05-20T14:33:23Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T14:33:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:33:23Z" name: success-200-isvc-31b2e-predictor-67cf4677b6-p2vdx.18b14ca6122a4be1 namespace: kserve-ci-e2e-test resourceVersion: "30592" uid: 8bae570d-9922-4493-8696-eec94f78d773 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:33:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-31b2e-predictor-67cf4677b6-p2vdx namespace: kserve-ci-e2e-test resourceVersion: "26475" uid: 1a64186b-ecb2-4088-9b56-24830d81d178 kind: Event lastTimestamp: "2026-05-20T14:33:23Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:33:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:33:23Z" name: success-200-isvc-31b2e-predictor-67cf4677b6-p2vdx.18b14ca6122ad178 namespace: kserve-ci-e2e-test resourceVersion: "30593" uid: 0b68a9d3-81f9-4943-b563-fdf131e20682 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:24:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-31b2e-predictor-67cf4677b6 namespace: kserve-ci-e2e-test resourceVersion: "26456" uid: e8669722-b851-4c22-98d7-ceed8bd53e32 kind: Event lastTimestamp: "2026-05-20T14:24:09Z" message: 'Created pod: success-200-isvc-31b2e-predictor-67cf4677b6-p2vdx' metadata: creationTimestamp: "2026-05-20T14:24:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:24:09Z" name: success-200-isvc-31b2e-predictor-67cf4677b6.18b14c24e2a52d2b namespace: kserve-ci-e2e-test resourceVersion: "26474" uid: 6da5fb00-0d51-4822-85ef-66b44a7af4df reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:24:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-31b2e-predictor namespace: kserve-ci-e2e-test resourceVersion: "26447" uid: e9e5c07f-bc2d-492f-b468-3cc3c4bf5c91 kind: Event lastTimestamp: "2026-05-20T14:24:09Z" message: Scaled up replica set success-200-isvc-31b2e-predictor-67cf4677b6 from 0 to 1 metadata: creationTimestamp: "2026-05-20T14:24:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:24:09Z" name: success-200-isvc-31b2e-predictor.18b14c24e074b189 namespace: kserve-ci-e2e-test resourceVersion: "26463" uid: eea5e998-8259-4a2f-b548-0eba0a392693 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:24:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-31b2e-predictor namespace: kserve-ci-e2e-test resourceVersion: "26453" uid: b440e4ee-053a-4555-bcb8-9f65ccf39d6a kind: Event lastTimestamp: "2026-05-20T14:24:24Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:24:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:24:24Z" name: success-200-isvc-31b2e-predictor.18b14c285eb309dd namespace: kserve-ci-e2e-test resourceVersion: "26695" uid: a6014a7d-ae04-430f-a577-9fdf2d672db0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:24:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-31b2e-predictor namespace: kserve-ci-e2e-test resourceVersion: "26453" uid: b440e4ee-053a-4555-bcb8-9f65ccf39d6a kind: Event lastTimestamp: "2026-05-20T14:24:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:24:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:24:24Z" name: success-200-isvc-31b2e-predictor.18b14c285f2eed60 namespace: kserve-ci-e2e-test resourceVersion: "26697" uid: 4e4f09d1-e257-4e3c-beeb-86348cd3d995 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-20T14:24:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-31b2e-predictor namespace: kserve-ci-e2e-test resourceVersion: "26696" uid: b440e4ee-053a-4555-bcb8-9f65ccf39d6a kind: Event lastTimestamp: "2026-05-20T14:25:24Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:24:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:25:24Z" name: success-200-isvc-31b2e-predictor.18b14c2bdea3f7a1 namespace: kserve-ci-e2e-test resourceVersion: "27271" uid: 389da325-ca12-4843-81ae-c95d6c383a86 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-20T14:24:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-31b2e-predictor namespace: kserve-ci-e2e-test resourceVersion: "26696" uid: b440e4ee-053a-4555-bcb8-9f65ccf39d6a kind: Event lastTimestamp: "2026-05-20T14:25:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:24:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:25:24Z" name: success-200-isvc-31b2e-predictor.18b14c2bdf24667f namespace: kserve-ci-e2e-test resourceVersion: "27272" uid: d9095234-6a5a-43c7-93a4-23eea0d9f8f5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-20T14:24:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-31b2e namespace: kserve-ci-e2e-test resourceVersion: "26438" uid: 6cf9a57c-4999-4f7d-861c-373417fa022b kind: Event lastTimestamp: "2026-05-20T14:24:58Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T14:24:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:24:58Z" name: success-200-isvc-31b2e.18b14c24dfe03025 namespace: kserve-ci-e2e-test resourceVersion: "27040" uid: 5d0d71e6-047d-4c61-9909-1cac6c27c582 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:24:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-31b2e namespace: kserve-ci-e2e-test resourceVersion: "26438" uid: 6cf9a57c-4999-4f7d-861c-373417fa022b kind: Event lastTimestamp: "2026-05-20T14:24:09Z" message: 'Failed to update status for InferenceService "success-200-isvc-31b2e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-31b2e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T14:24:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:24:09Z" name: success-200-isvc-31b2e.18b14c24e0ae8be6 namespace: kserve-ci-e2e-test resourceVersion: "26464" uid: 44979add-bd6d-4b60-8852-ce21d59e6224 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:24:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-31b2e namespace: kserve-ci-e2e-test resourceVersion: "26438" uid: 6cf9a57c-4999-4f7d-861c-373417fa022b kind: Event lastTimestamp: "2026-05-20T14:24:09Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-31b2e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T14:24:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:24:09Z" name: success-200-isvc-31b2e.18b14c24e0aef984 namespace: kserve-ci-e2e-test resourceVersion: "26468" uid: ca374a90-e2ef-4874-9f04-9bfe8f02c364 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:24:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-31b2e namespace: kserve-ci-e2e-test resourceVersion: "27036" uid: 6cf9a57c-4999-4f7d-861c-373417fa022b kind: Event lastTimestamp: "2026-05-20T14:24:57Z" message: InferenceService [success-200-isvc-31b2e] is Ready metadata: creationTimestamp: "2026-05-20T14:24:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:24:57Z" name: success-200-isvc-31b2e.18b14c3044aa594b namespace: kserve-ci-e2e-test resourceVersion: "27037" uid: ead46ea3-0522-4b67-aabc-a19f8515398d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T14:23:28.626484Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3e303-predictor-54ff64d9f6-79pgj namespace: kserve-ci-e2e-test resourceVersion: "25660" uid: e1008ccb-006b-427d-974d-c62139642892 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-3e303-predictor-54ff64d9f6-79pgj to ip-10-0-137-148.ec2.internal metadata: creationTimestamp: "2026-05-20T14:23:28Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T14:23:28Z" name: success-200-isvc-3e303-predictor-54ff64d9f6-79pgj.18b14c1b76a64f7c namespace: kserve-ci-e2e-test resourceVersion: "25664" uid: 20544155-03ef-46ab-a46b-f30d99637b31 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-747dc7bb84-jx2kx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:23:29Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3e303-predictor-54ff64d9f6-79pgj namespace: kserve-ci-e2e-test resourceVersion: "25669" uid: e1008ccb-006b-427d-974d-c62139642892 kind: Event lastTimestamp: "2026-05-20T14:23:29Z" message: Add eth0 [10.133.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T14:23:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T14:23:29Z" name: success-200-isvc-3e303-predictor-54ff64d9f6-79pgj.18b14c1b8fdec31d namespace: kserve-ci-e2e-test resourceVersion: "25710" uid: 11bbdb9a-08bc-4637-b904-dc59f86fa181 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:23:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3e303-predictor-54ff64d9f6-79pgj namespace: kserve-ci-e2e-test resourceVersion: "25661" uid: e1008ccb-006b-427d-974d-c62139642892 kind: Event lastTimestamp: "2026-05-20T14:23:29Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1501" already present on machine metadata: creationTimestamp: "2026-05-20T14:23:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:23:29Z" name: success-200-isvc-3e303-predictor-54ff64d9f6-79pgj.18b14c1b91680a3e namespace: kserve-ci-e2e-test resourceVersion: "25713" uid: eb68e5fe-121a-411c-af5a-261742e78ed8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:23:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3e303-predictor-54ff64d9f6-79pgj namespace: kserve-ci-e2e-test resourceVersion: "25661" uid: e1008ccb-006b-427d-974d-c62139642892 kind: Event lastTimestamp: "2026-05-20T14:23:29Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T14:23:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:23:29Z" name: success-200-isvc-3e303-predictor-54ff64d9f6-79pgj.18b14c1b945776c3 namespace: kserve-ci-e2e-test resourceVersion: "25714" uid: fa46a30a-4f24-4d48-9199-76b32231af81 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:23:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3e303-predictor-54ff64d9f6-79pgj namespace: kserve-ci-e2e-test resourceVersion: "25661" uid: e1008ccb-006b-427d-974d-c62139642892 kind: Event lastTimestamp: "2026-05-20T14:23:29Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T14:23:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:23:29Z" name: success-200-isvc-3e303-predictor-54ff64d9f6-79pgj.18b14c1b94c402d7 namespace: kserve-ci-e2e-test resourceVersion: "25715" uid: a695238c-5449-4b0b-a1a7-90eeea30dcac reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:23:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3e303-predictor-54ff64d9f6-79pgj namespace: kserve-ci-e2e-test resourceVersion: "25661" uid: e1008ccb-006b-427d-974d-c62139642892 kind: Event lastTimestamp: "2026-05-20T14:23:29Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-20T14:23:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:23:29Z" name: success-200-isvc-3e303-predictor-54ff64d9f6-79pgj.18b14c1b94ceff14 namespace: kserve-ci-e2e-test resourceVersion: "25716" uid: 97ca8562-4d46-451d-ab6f-03321ca90006 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:23:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3e303-predictor-54ff64d9f6-79pgj namespace: kserve-ci-e2e-test resourceVersion: "25661" uid: e1008ccb-006b-427d-974d-c62139642892 kind: Event lastTimestamp: "2026-05-20T14:23:29Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T14:23:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:23:29Z" name: success-200-isvc-3e303-predictor-54ff64d9f6-79pgj.18b14c1b97a87893 namespace: kserve-ci-e2e-test resourceVersion: "25717" uid: fbeea8d9-db1c-40ee-8dc0-a85a79758fe8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:23:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3e303-predictor-54ff64d9f6-79pgj namespace: kserve-ci-e2e-test resourceVersion: "25661" uid: e1008ccb-006b-427d-974d-c62139642892 kind: Event lastTimestamp: "2026-05-20T14:23:29Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:23:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:23:29Z" name: success-200-isvc-3e303-predictor-54ff64d9f6-79pgj.18b14c1b98150c3d namespace: kserve-ci-e2e-test resourceVersion: "25718" uid: 820d6efa-f098-44b7-bd20-390676f2470a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-20T14:23:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3e303-predictor-54ff64d9f6-79pgj namespace: kserve-ci-e2e-test resourceVersion: "25661" uid: e1008ccb-006b-427d-974d-c62139642892 kind: Event lastTimestamp: "2026-05-20T14:24:05Z" message: 'Readiness probe failed: dial tcp 10.133.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T14:23:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:24:05Z" name: success-200-isvc-3e303-predictor-54ff64d9f6-79pgj.18b14c1bb91b91e0 namespace: kserve-ci-e2e-test resourceVersion: "26384" uid: e4649ea7-45d6-4729-8c05-1987a61a3831 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:40:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3e303-predictor-54ff64d9f6-79pgj namespace: kserve-ci-e2e-test resourceVersion: "25661" uid: e1008ccb-006b-427d-974d-c62139642892 kind: Event lastTimestamp: "2026-05-20T14:40:48Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T14:40:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:40:48Z" name: success-200-isvc-3e303-predictor-54ff64d9f6-79pgj.18b14d0d7b00ef9d namespace: kserve-ci-e2e-test resourceVersion: "33660" uid: fe7e6592-9305-4dcd-8ffd-406a315fbd86 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:40:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3e303-predictor-54ff64d9f6-79pgj namespace: kserve-ci-e2e-test resourceVersion: "25661" uid: e1008ccb-006b-427d-974d-c62139642892 kind: Event lastTimestamp: "2026-05-20T14:40:48Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:40:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:40:48Z" name: success-200-isvc-3e303-predictor-54ff64d9f6-79pgj.18b14d0d7b016b88 namespace: kserve-ci-e2e-test resourceVersion: "33661" uid: 5d4fcda1-df41-4379-931a-376a9738ab3c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:40:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3e303-predictor-54ff64d9f6-79pgj namespace: kserve-ci-e2e-test resourceVersion: "25661" uid: e1008ccb-006b-427d-974d-c62139642892 kind: Event lastTimestamp: "2026-05-20T14:40:50Z" message: 'Readiness probe failed: Get "https://10.133.0.29:8643/healthz": dial tcp 10.133.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-20T14:40:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:40:50Z" name: success-200-isvc-3e303-predictor-54ff64d9f6-79pgj.18b14d0e19b4625f namespace: kserve-ci-e2e-test resourceVersion: "33725" uid: 1a44c485-5469-46a7-9d3a-61b1c0fe965e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:23:28Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-3e303-predictor-54ff64d9f6 namespace: kserve-ci-e2e-test resourceVersion: "25648" uid: 0f3674d4-5484-48d1-98fb-ba8bc311bd37 kind: Event lastTimestamp: "2026-05-20T14:23:28Z" message: 'Created pod: success-200-isvc-3e303-predictor-54ff64d9f6-79pgj' metadata: creationTimestamp: "2026-05-20T14:23:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:23:28Z" name: success-200-isvc-3e303-predictor-54ff64d9f6.18b14c1b762b25cf namespace: kserve-ci-e2e-test resourceVersion: "25662" uid: f224dd37-05bd-4890-b463-64c978b68f57 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:23:28Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-3e303-predictor namespace: kserve-ci-e2e-test resourceVersion: "25638" uid: 4a6c2581-44c1-45d8-9d42-91c27ff52876 kind: Event lastTimestamp: "2026-05-20T14:23:28Z" message: Scaled up replica set success-200-isvc-3e303-predictor-54ff64d9f6 from 0 to 1 metadata: creationTimestamp: "2026-05-20T14:23:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:23:28Z" name: success-200-isvc-3e303-predictor.18b14c1b73d5a9a7 namespace: kserve-ci-e2e-test resourceVersion: "25653" uid: fda3899a-70bc-457e-b1c1-026072c195f2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T14:23:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3e303-predictor namespace: kserve-ci-e2e-test resourceVersion: "25647" uid: a2a49f08-1e9f-47c5-9392-7c90ad8ea14f kind: Event lastTimestamp: "2026-05-20T14:23:58Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:23:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:23:58Z" name: success-200-isvc-3e303-predictor.18b14c1ef29aebcd namespace: kserve-ci-e2e-test resourceVersion: "26285" uid: 2d9e1e40-70e3-47af-8566-f4d8d26ee147 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T14:23:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3e303-predictor namespace: kserve-ci-e2e-test resourceVersion: "25647" uid: a2a49f08-1e9f-47c5-9392-7c90ad8ea14f kind: Event lastTimestamp: "2026-05-20T14:23:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:23:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:23:58Z" name: success-200-isvc-3e303-predictor.18b14c1ef30cc957 namespace: kserve-ci-e2e-test resourceVersion: "26286" uid: 2ea25d61-13e6-4a6e-aa54-9d0dfae4b635 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T14:24:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3e303-predictor namespace: kserve-ci-e2e-test resourceVersion: "26069" uid: a2a49f08-1e9f-47c5-9392-7c90ad8ea14f kind: Event lastTimestamp: "2026-05-20T14:24:43Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:24:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:24:43Z" name: success-200-isvc-3e303-predictor.18b14c25f0f1e593 namespace: kserve-ci-e2e-test resourceVersion: "26910" uid: c1732816-4eb9-480b-8336-706e54cccca6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T14:24:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3e303-predictor namespace: kserve-ci-e2e-test resourceVersion: "26069" uid: a2a49f08-1e9f-47c5-9392-7c90ad8ea14f kind: Event lastTimestamp: "2026-05-20T14:24:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:24:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:24:43Z" name: success-200-isvc-3e303-predictor.18b14c25f1666013 namespace: kserve-ci-e2e-test resourceVersion: "26911" uid: a4e7e3a4-85b4-4bf5-907f-cace54355386 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-20T14:23:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3e303 namespace: kserve-ci-e2e-test resourceVersion: "25614" uid: 86dff003-8225-4b6c-8aeb-808439104a32 kind: Event lastTimestamp: "2026-05-20T14:24:15Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T14:23:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:24:15Z" name: success-200-isvc-3e303.18b14c1b735cad5e namespace: kserve-ci-e2e-test resourceVersion: "26629" uid: b88399ee-3782-4fd1-9ceb-312e18d98ca6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:23:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3e303 namespace: kserve-ci-e2e-test resourceVersion: "25614" uid: 86dff003-8225-4b6c-8aeb-808439104a32 kind: Event lastTimestamp: "2026-05-20T14:23:28Z" message: 'Failed to update status for InferenceService "success-200-isvc-3e303": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-3e303": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T14:23:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:23:28Z" name: success-200-isvc-3e303.18b14c1b7475de13 namespace: kserve-ci-e2e-test resourceVersion: "25654" uid: 6cab3cf8-7531-498b-a8f0-004cf4052d09 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:23:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3e303 namespace: kserve-ci-e2e-test resourceVersion: "25614" uid: 86dff003-8225-4b6c-8aeb-808439104a32 kind: Event lastTimestamp: "2026-05-20T14:23:28Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-3e303": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T14:23:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:23:28Z" name: success-200-isvc-3e303.18b14c1b74762008 namespace: kserve-ci-e2e-test resourceVersion: "25657" uid: 53369981-29ec-4abc-8c5a-7d56eb59f2c2 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:24:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3e303 namespace: kserve-ci-e2e-test resourceVersion: "26625" uid: 86dff003-8225-4b6c-8aeb-808439104a32 kind: Event lastTimestamp: "2026-05-20T14:24:15Z" message: InferenceService [success-200-isvc-3e303] is Ready metadata: creationTimestamp: "2026-05-20T14:24:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:24:15Z" name: success-200-isvc-3e303.18b14c267598016a namespace: kserve-ci-e2e-test resourceVersion: "26627" uid: ad2cac8a-b4d3-4fb2-ac37-ae1546c9ef7a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T14:01:38.297822Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-69925-predictor-697c88cf8-dk2sh namespace: kserve-ci-e2e-test resourceVersion: "14538" uid: eacdadf1-fcb3-46b2-b379-1cf4e772e64f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-69925-predictor-697c88cf8-dk2sh to ip-10-0-137-148.ec2.internal metadata: creationTimestamp: "2026-05-20T14:01:38Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T14:01:38Z" name: success-200-isvc-69925-predictor-697c88cf8-dk2sh.18b14aea60f98f39 namespace: kserve-ci-e2e-test resourceVersion: "14545" uid: 346274da-982b-4034-b225-45a6fd3f6173 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-747dc7bb84-jx2kx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:38Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-69925-predictor-697c88cf8-dk2sh namespace: kserve-ci-e2e-test resourceVersion: "14551" uid: eacdadf1-fcb3-46b2-b379-1cf4e772e64f kind: Event lastTimestamp: "2026-05-20T14:01:38Z" message: Add eth0 [10.133.0.19/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T14:01:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T14:01:38Z" name: success-200-isvc-69925-predictor-697c88cf8-dk2sh.18b14aea7a737718 namespace: kserve-ci-e2e-test resourceVersion: "14607" uid: 2ab1cf4f-98de-47db-9988-223b00379ab5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-69925-predictor-697c88cf8-dk2sh namespace: kserve-ci-e2e-test resourceVersion: "14541" uid: eacdadf1-fcb3-46b2-b379-1cf4e772e64f kind: Event lastTimestamp: "2026-05-20T14:01:38Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1501" metadata: creationTimestamp: "2026-05-20T14:01:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:38Z" name: success-200-isvc-69925-predictor-697c88cf8-dk2sh.18b14aea7df7b1b1 namespace: kserve-ci-e2e-test resourceVersion: "14609" uid: 58fae208-f304-4762-9863-76a5775faea0 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-69925-predictor-697c88cf8-dk2sh namespace: kserve-ci-e2e-test resourceVersion: "14541" uid: eacdadf1-fcb3-46b2-b379-1cf4e772e64f kind: Event lastTimestamp: "2026-05-20T14:01:51Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1501" in 12.301s (12.301s including waiting). Image size: 1335711840 bytes.' metadata: creationTimestamp: "2026-05-20T14:01:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:51Z" name: success-200-isvc-69925-predictor-697c88cf8-dk2sh.18b14aed5b31fc7b namespace: kserve-ci-e2e-test resourceVersion: "14780" uid: 817d9756-0b15-4c54-bd6d-fd1cd68ffb94 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-69925-predictor-697c88cf8-dk2sh namespace: kserve-ci-e2e-test resourceVersion: "14541" uid: eacdadf1-fcb3-46b2-b379-1cf4e772e64f kind: Event lastTimestamp: "2026-05-20T14:01:51Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T14:01:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:51Z" name: success-200-isvc-69925-predictor-697c88cf8-dk2sh.18b14aed5f936a2b namespace: kserve-ci-e2e-test resourceVersion: "14781" uid: 4a7fa5c2-6375-4ada-a6f5-f536cb42d38a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-69925-predictor-697c88cf8-dk2sh namespace: kserve-ci-e2e-test resourceVersion: "14541" uid: eacdadf1-fcb3-46b2-b379-1cf4e772e64f kind: Event lastTimestamp: "2026-05-20T14:01:51Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T14:01:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:51Z" name: success-200-isvc-69925-predictor-697c88cf8-dk2sh.18b14aed602cae21 namespace: kserve-ci-e2e-test resourceVersion: "14782" uid: cd6058d9-a745-4a6e-990b-857803e69d1e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-69925-predictor-697c88cf8-dk2sh namespace: kserve-ci-e2e-test resourceVersion: "14541" uid: eacdadf1-fcb3-46b2-b379-1cf4e772e64f kind: Event lastTimestamp: "2026-05-20T14:01:51Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-05-20T14:01:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:51Z" name: success-200-isvc-69925-predictor-697c88cf8-dk2sh.18b14aed603d1d61 namespace: kserve-ci-e2e-test resourceVersion: "14783" uid: 097c3f20-aa98-4ed1-a162-161d8cdbea3d reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-69925-predictor-697c88cf8-dk2sh namespace: kserve-ci-e2e-test resourceVersion: "14541" uid: eacdadf1-fcb3-46b2-b379-1cf4e772e64f kind: Event lastTimestamp: "2026-05-20T14:01:58Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 7.337s (7.337s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-05-20T14:01:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:58Z" name: success-200-isvc-69925-predictor-697c88cf8-dk2sh.18b14aef1597ee15 namespace: kserve-ci-e2e-test resourceVersion: "14866" uid: 10fa3f3b-2fcc-4e1e-878f-26a5fb698a0c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-69925-predictor-697c88cf8-dk2sh namespace: kserve-ci-e2e-test resourceVersion: "14541" uid: eacdadf1-fcb3-46b2-b379-1cf4e772e64f kind: Event lastTimestamp: "2026-05-20T14:01:58Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T14:01:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:58Z" name: success-200-isvc-69925-predictor-697c88cf8-dk2sh.18b14aef19d729a7 namespace: kserve-ci-e2e-test resourceVersion: "14871" uid: a59a60ac-450a-49d2-8560-c89a7e337938 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-69925-predictor-697c88cf8-dk2sh namespace: kserve-ci-e2e-test resourceVersion: "14541" uid: eacdadf1-fcb3-46b2-b379-1cf4e772e64f kind: Event lastTimestamp: "2026-05-20T14:01:58Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:01:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:58Z" name: success-200-isvc-69925-predictor-697c88cf8-dk2sh.18b14aef1ab6a448 namespace: kserve-ci-e2e-test resourceVersion: "14872" uid: 112260a6-8685-47e5-91c3-b8863ed34887 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-05-20T14:02:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-69925-predictor-697c88cf8-dk2sh namespace: kserve-ci-e2e-test resourceVersion: "14541" uid: eacdadf1-fcb3-46b2-b379-1cf4e772e64f kind: Event lastTimestamp: "2026-05-20T14:02:46Z" message: 'Readiness probe failed: dial tcp 10.133.0.19:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T14:02:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:02:46Z" name: success-200-isvc-69925-predictor-697c88cf8-dk2sh.18b14aef70c37498 namespace: kserve-ci-e2e-test resourceVersion: "15280" uid: cff74d49-c71e-49f3-a0bf-e40ab4d4edf5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-69925-predictor-697c88cf8-dk2sh namespace: kserve-ci-e2e-test resourceVersion: "14541" uid: eacdadf1-fcb3-46b2-b379-1cf4e772e64f kind: Event lastTimestamp: "2026-05-20T14:03:18Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T14:03:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:18Z" name: success-200-isvc-69925-predictor-697c88cf8-dk2sh.18b14b01b6644208 namespace: kserve-ci-e2e-test resourceVersion: "15655" uid: 7a0f57c9-8648-4160-b070-a85e9e4b3e4d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-69925-predictor-697c88cf8-dk2sh namespace: kserve-ci-e2e-test resourceVersion: "14541" uid: eacdadf1-fcb3-46b2-b379-1cf4e772e64f kind: Event lastTimestamp: "2026-05-20T14:03:18Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:03:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:18Z" name: success-200-isvc-69925-predictor-697c88cf8-dk2sh.18b14b01b664d083 namespace: kserve-ci-e2e-test resourceVersion: "15657" uid: 492b1c15-1894-45a5-9c7c-cc857f20a0f0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-69925-predictor-697c88cf8-dk2sh namespace: kserve-ci-e2e-test resourceVersion: "14541" uid: eacdadf1-fcb3-46b2-b379-1cf4e772e64f kind: Event lastTimestamp: "2026-05-20T14:03:21Z" message: 'Readiness probe failed: Get "https://10.133.0.19:8643/healthz": dial tcp 10.133.0.19:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-20T14:03:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:21Z" name: success-200-isvc-69925-predictor-697c88cf8-dk2sh.18b14b024cdc2ece namespace: kserve-ci-e2e-test resourceVersion: "15773" uid: fe5b4243-7953-46d9-bad6-f7fdd4ff3b4d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:38Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-69925-predictor-697c88cf8 namespace: kserve-ci-e2e-test resourceVersion: "14532" uid: 607c2955-0005-4428-a770-9f65278490a4 kind: Event lastTimestamp: "2026-05-20T14:01:38Z" message: 'Created pod: success-200-isvc-69925-predictor-697c88cf8-dk2sh' metadata: creationTimestamp: "2026-05-20T14:01:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:01:38Z" name: success-200-isvc-69925-predictor-697c88cf8.18b14aea604fecee namespace: kserve-ci-e2e-test resourceVersion: "14542" uid: 24f65af7-85b2-4f57-9b1d-ab317a686590 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:38Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-69925-predictor namespace: kserve-ci-e2e-test resourceVersion: "14529" uid: 24dffcbf-7e7a-4ddb-b4ee-53cd9d3a9f7e kind: Event lastTimestamp: "2026-05-20T14:01:38Z" message: Scaled up replica set success-200-isvc-69925-predictor-697c88cf8 from 0 to 1 metadata: creationTimestamp: "2026-05-20T14:01:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:01:38Z" name: success-200-isvc-69925-predictor.18b14aea5e3bb489 namespace: kserve-ci-e2e-test resourceVersion: "14535" uid: c17d621d-9f11-4d40-b776-49485029f28f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T14:01:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-69925-predictor namespace: kserve-ci-e2e-test resourceVersion: "14531" uid: 0c7f1068-b952-48a4-a055-98c592095c8e kind: Event lastTimestamp: "2026-05-20T14:02:23Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:01:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:02:23Z" name: success-200-isvc-69925-predictor.18b14aeddcfb89fd namespace: kserve-ci-e2e-test resourceVersion: "15089" uid: 5a26190c-8ddb-4df5-bd0d-aa63219ab8dd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T14:01:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-69925-predictor namespace: kserve-ci-e2e-test resourceVersion: "14531" uid: 0c7f1068-b952-48a4-a055-98c592095c8e kind: Event lastTimestamp: "2026-05-20T14:02:23Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:01:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:02:23Z" name: success-200-isvc-69925-predictor.18b14aeddd6badb0 namespace: kserve-ci-e2e-test resourceVersion: "15090" uid: 469e6752-f767-4e47-8906-4e81d0915ad9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T14:02:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-69925-predictor namespace: kserve-ci-e2e-test resourceVersion: "14808" uid: 0c7f1068-b952-48a4-a055-98c592095c8e kind: Event lastTimestamp: "2026-05-20T14:03:08Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:02:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:03:08Z" name: success-200-isvc-69925-predictor.18b14af85a81b08c namespace: kserve-ci-e2e-test resourceVersion: "15490" uid: 069e2887-6af0-490a-8003-ed993f477898 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T14:02:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-69925-predictor namespace: kserve-ci-e2e-test resourceVersion: "14808" uid: 0c7f1068-b952-48a4-a055-98c592095c8e kind: Event lastTimestamp: "2026-05-20T14:03:08Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:02:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:03:08Z" name: success-200-isvc-69925-predictor.18b14af85b1ddbcc namespace: kserve-ci-e2e-test resourceVersion: "15491" uid: 4a73acdb-cca8-44e9-8f82-a3a35598d5ca reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-20T14:01:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-69925 namespace: kserve-ci-e2e-test resourceVersion: "14521" uid: 91e34197-25ad-4ba0-8a94-d6f3258f72af kind: Event lastTimestamp: "2026-05-20T14:02:56Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T14:01:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:02:56Z" name: success-200-isvc-69925.18b14aea5e077e2c namespace: kserve-ci-e2e-test resourceVersion: "15379" uid: 2a85fa4b-ba3e-445b-bfaa-be8902b937ef reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:02:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-69925 namespace: kserve-ci-e2e-test resourceVersion: "15375" uid: 91e34197-25ad-4ba0-8a94-d6f3258f72af kind: Event lastTimestamp: "2026-05-20T14:02:56Z" message: InferenceService [success-200-isvc-69925] is Ready metadata: creationTimestamp: "2026-05-20T14:02:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:02:56Z" name: success-200-isvc-69925.18b14afc849f4634 namespace: kserve-ci-e2e-test resourceVersion: "15377" uid: 21e70435-b32c-4009-9caa-9f74d1d96ee2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T14:14:13.684993Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-734fc-predictor-64d98dd8b6-bknhg namespace: kserve-ci-e2e-test resourceVersion: "21356" uid: d70962d5-e2e1-45e6-b610-376b13dc6648 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-734fc-predictor-64d98dd8b6-bknhg to ip-10-0-137-148.ec2.internal metadata: creationTimestamp: "2026-05-20T14:14:13Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T14:14:13Z" name: success-200-isvc-734fc-predictor-64d98dd8b6-bknhg.18b14b9a418f1217 namespace: kserve-ci-e2e-test resourceVersion: "21364" uid: 2b0ac452-7cae-4dc3-928a-d31fa7fe5626 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-747dc7bb84-jx2kx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:14:13Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-734fc-predictor-64d98dd8b6-bknhg namespace: kserve-ci-e2e-test resourceVersion: "21361" uid: d70962d5-e2e1-45e6-b610-376b13dc6648 kind: Event lastTimestamp: "2026-05-20T14:14:13Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-734fc-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-20T14:14:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:14:13Z" name: success-200-isvc-734fc-predictor-64d98dd8b6-bknhg.18b14b9a4c890f2e namespace: kserve-ci-e2e-test resourceVersion: "21404" uid: e17cc263-ad82-422c-a777-3249a76a4f44 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:14:14Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-734fc-predictor-64d98dd8b6-bknhg namespace: kserve-ci-e2e-test resourceVersion: "21373" uid: d70962d5-e2e1-45e6-b610-376b13dc6648 kind: Event lastTimestamp: "2026-05-20T14:14:14Z" message: Add eth0 [10.133.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T14:14:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T14:14:14Z" name: success-200-isvc-734fc-predictor-64d98dd8b6-bknhg.18b14b9a7e66314c namespace: kserve-ci-e2e-test resourceVersion: "21427" uid: 11e26f6a-4ef2-4205-a418-afcf78ef262b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:14:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-734fc-predictor-64d98dd8b6-bknhg namespace: kserve-ci-e2e-test resourceVersion: "21361" uid: d70962d5-e2e1-45e6-b610-376b13dc6648 kind: Event lastTimestamp: "2026-05-20T14:14:14Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1501" already present on machine metadata: creationTimestamp: "2026-05-20T14:14:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:14:14Z" name: success-200-isvc-734fc-predictor-64d98dd8b6-bknhg.18b14b9a7fbd7aa8 namespace: kserve-ci-e2e-test resourceVersion: "21430" uid: f3a42a99-f1e5-4fb2-966b-c23145a92d49 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:14:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-734fc-predictor-64d98dd8b6-bknhg namespace: kserve-ci-e2e-test resourceVersion: "21361" uid: d70962d5-e2e1-45e6-b610-376b13dc6648 kind: Event lastTimestamp: "2026-05-20T14:14:14Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T14:14:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:14:14Z" name: success-200-isvc-734fc-predictor-64d98dd8b6-bknhg.18b14b9a832110d8 namespace: kserve-ci-e2e-test resourceVersion: "21431" uid: 2e73f001-a82d-4df4-b981-1d9d5a4b8b57 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:14:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-734fc-predictor-64d98dd8b6-bknhg namespace: kserve-ci-e2e-test resourceVersion: "21361" uid: d70962d5-e2e1-45e6-b610-376b13dc6648 kind: Event lastTimestamp: "2026-05-20T14:14:14Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T14:14:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:14:14Z" name: success-200-isvc-734fc-predictor-64d98dd8b6-bknhg.18b14b9a838d702f namespace: kserve-ci-e2e-test resourceVersion: "21432" uid: cb7013f5-2f39-4f89-855c-4341bb9ea2d6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:14:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-734fc-predictor-64d98dd8b6-bknhg namespace: kserve-ci-e2e-test resourceVersion: "21361" uid: d70962d5-e2e1-45e6-b610-376b13dc6648 kind: Event lastTimestamp: "2026-05-20T14:14:14Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-20T14:14:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:14:14Z" name: success-200-isvc-734fc-predictor-64d98dd8b6-bknhg.18b14b9a83989f8f namespace: kserve-ci-e2e-test resourceVersion: "21434" uid: 9b2abfdf-09d7-4cd0-955d-86a71afff420 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:14:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-734fc-predictor-64d98dd8b6-bknhg namespace: kserve-ci-e2e-test resourceVersion: "21361" uid: d70962d5-e2e1-45e6-b610-376b13dc6648 kind: Event lastTimestamp: "2026-05-20T14:14:14Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T14:14:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:14:14Z" name: success-200-isvc-734fc-predictor-64d98dd8b6-bknhg.18b14b9a866e40a1 namespace: kserve-ci-e2e-test resourceVersion: "21435" uid: 55d39d3a-8616-4df4-af00-a4b6036bc422 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:14:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-734fc-predictor-64d98dd8b6-bknhg namespace: kserve-ci-e2e-test resourceVersion: "21361" uid: d70962d5-e2e1-45e6-b610-376b13dc6648 kind: Event lastTimestamp: "2026-05-20T14:14:14Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:14:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:14:14Z" name: success-200-isvc-734fc-predictor-64d98dd8b6-bknhg.18b14b9a870e519f namespace: kserve-ci-e2e-test resourceVersion: "21436" uid: da833f5b-8c69-49c9-91b1-9520acb478f3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-20T14:14:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-734fc-predictor-64d98dd8b6-bknhg namespace: kserve-ci-e2e-test resourceVersion: "21361" uid: d70962d5-e2e1-45e6-b610-376b13dc6648 kind: Event lastTimestamp: "2026-05-20T14:14:52Z" message: 'Readiness probe failed: dial tcp 10.133.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T14:14:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:14:52Z" name: success-200-isvc-734fc-predictor-64d98dd8b6-bknhg.18b14b9ad70fe3dd namespace: kserve-ci-e2e-test resourceVersion: "21813" uid: 3dcadc55-6531-4392-844a-ebcce15e5c4e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:23:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-734fc-predictor-64d98dd8b6-bknhg namespace: kserve-ci-e2e-test resourceVersion: "21361" uid: d70962d5-e2e1-45e6-b610-376b13dc6648 kind: Event lastTimestamp: "2026-05-20T14:23:28Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T14:23:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:23:28Z" name: success-200-isvc-734fc-predictor-64d98dd8b6-bknhg.18b14c1b74d8762b namespace: kserve-ci-e2e-test resourceVersion: "25656" uid: 46672440-e556-44e3-858c-7ad36e6cfe8b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:23:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-734fc-predictor-64d98dd8b6-bknhg namespace: kserve-ci-e2e-test resourceVersion: "21361" uid: d70962d5-e2e1-45e6-b610-376b13dc6648 kind: Event lastTimestamp: "2026-05-20T14:23:28Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:23:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:23:28Z" name: success-200-isvc-734fc-predictor-64d98dd8b6-bknhg.18b14c1b74d90a6d namespace: kserve-ci-e2e-test resourceVersion: "25658" uid: 31b134d1-f47d-438d-95ab-745917002da2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:14:13Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-734fc-predictor-64d98dd8b6 namespace: kserve-ci-e2e-test resourceVersion: "21342" uid: 0a4a59e8-5cf0-4f38-bb1d-76d66300ec3c kind: Event lastTimestamp: "2026-05-20T14:14:13Z" message: 'Created pod: success-200-isvc-734fc-predictor-64d98dd8b6-bknhg' metadata: creationTimestamp: "2026-05-20T14:14:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:14:13Z" name: success-200-isvc-734fc-predictor-64d98dd8b6.18b14b9a4104bb2e namespace: kserve-ci-e2e-test resourceVersion: "21359" uid: ad03ff15-4f41-4c7f-8461-db7b3ecb82f8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:14:13Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-734fc-predictor namespace: kserve-ci-e2e-test resourceVersion: "21339" uid: c89e0a87-2c3b-4b4f-bc61-58f42f32b14f kind: Event lastTimestamp: "2026-05-20T14:14:13Z" message: Scaled up replica set success-200-isvc-734fc-predictor-64d98dd8b6 from 0 to 1 metadata: creationTimestamp: "2026-05-20T14:14:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:14:13Z" name: success-200-isvc-734fc-predictor.18b14b9a3e9d06db namespace: kserve-ci-e2e-test resourceVersion: "21345" uid: 58ea8b39-6a37-4a47-86e7-c598058e0f1a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T14:14:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-734fc-predictor namespace: kserve-ci-e2e-test resourceVersion: "21341" uid: 82085ba1-cc40-46a3-90ad-ab04c426928d kind: Event lastTimestamp: "2026-05-20T14:14:43Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:14:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:14:43Z" name: success-200-isvc-734fc-predictor.18b14b9dbe1c36a0 namespace: kserve-ci-e2e-test resourceVersion: "21705" uid: e4f6891b-0934-4c98-b001-d973405a087e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T14:14:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-734fc-predictor namespace: kserve-ci-e2e-test resourceVersion: "21341" uid: 82085ba1-cc40-46a3-90ad-ab04c426928d kind: Event lastTimestamp: "2026-05-20T14:14:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:14:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:14:43Z" name: success-200-isvc-734fc-predictor.18b14b9dbf2bee4c namespace: kserve-ci-e2e-test resourceVersion: "21706" uid: 6c22bb11-467f-492e-a8cb-aada4edd2bbe reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-20T14:14:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-734fc-predictor namespace: kserve-ci-e2e-test resourceVersion: "21560" uid: 82085ba1-cc40-46a3-90ad-ab04c426928d kind: Event lastTimestamp: "2026-05-20T14:15:43Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:14:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:15:43Z" name: success-200-isvc-734fc-predictor.18b14ba4bda9c7d2 namespace: kserve-ci-e2e-test resourceVersion: "22262" uid: fbbe7145-589d-4251-b896-b3fc49b60060 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-20T14:14:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-734fc-predictor namespace: kserve-ci-e2e-test resourceVersion: "21560" uid: 82085ba1-cc40-46a3-90ad-ab04c426928d kind: Event lastTimestamp: "2026-05-20T14:15:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:14:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:15:43Z" name: success-200-isvc-734fc-predictor.18b14ba4be25f695 namespace: kserve-ci-e2e-test resourceVersion: "22263" uid: 27c2fe20-f8f0-4f6b-905b-865b2d80ca6d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-20T14:14:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-734fc namespace: kserve-ci-e2e-test resourceVersion: "21320" uid: 9dea9640-ae92-4742-bb55-f708e192ad11 kind: Event lastTimestamp: "2026-05-20T14:15:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T14:14:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:15:02Z" name: success-200-isvc-734fc.18b14b9a3e62be90 namespace: kserve-ci-e2e-test resourceVersion: "21913" uid: 8d0cdc70-37d5-4205-8850-ed4b4451f93c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:14:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-734fc namespace: kserve-ci-e2e-test resourceVersion: "21320" uid: 9dea9640-ae92-4742-bb55-f708e192ad11 kind: Event lastTimestamp: "2026-05-20T14:14:13Z" message: 'Failed to update status for InferenceService "success-200-isvc-734fc": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-734fc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T14:14:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:14:13Z" name: success-200-isvc-734fc.18b14b9a3eec51f8 namespace: kserve-ci-e2e-test resourceVersion: "21348" uid: 708ed0ec-d16e-44d2-9e50-c465f4ccc016 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:14:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-734fc namespace: kserve-ci-e2e-test resourceVersion: "21320" uid: 9dea9640-ae92-4742-bb55-f708e192ad11 kind: Event lastTimestamp: "2026-05-20T14:14:13Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-734fc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T14:14:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:14:13Z" name: success-200-isvc-734fc.18b14b9a3eeca77c namespace: kserve-ci-e2e-test resourceVersion: "21350" uid: 2b4949e3-f458-4115-af01-f81ae3f31999 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:15:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-734fc namespace: kserve-ci-e2e-test resourceVersion: "21906" uid: 9dea9640-ae92-4742-bb55-f708e192ad11 kind: Event lastTimestamp: "2026-05-20T14:15:02Z" message: InferenceService [success-200-isvc-734fc] is Ready metadata: creationTimestamp: "2026-05-20T14:15:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:15:02Z" name: success-200-isvc-734fc.18b14ba5958d095e namespace: kserve-ci-e2e-test resourceVersion: "21909" uid: cc7f3b28-3a69-4328-8401-6717bf541335 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T14:03:18.578668Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-83116-predictor-59d5fb4b86-m27ms namespace: kserve-ci-e2e-test resourceVersion: "15660" uid: 5b59113e-1550-4804-b847-d4e3d67f7931 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-83116-predictor-59d5fb4b86-m27ms to ip-10-0-137-148.ec2.internal metadata: creationTimestamp: "2026-05-20T14:03:18Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T14:03:18Z" name: success-200-isvc-83116-predictor-59d5fb4b86-m27ms.18b14b01ba2deeef namespace: kserve-ci-e2e-test resourceVersion: "15665" uid: 80710f21-31cb-4192-a674-8516519a1c58 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-747dc7bb84-jx2kx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:18Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-83116-predictor-59d5fb4b86-m27ms namespace: kserve-ci-e2e-test resourceVersion: "15663" uid: 5b59113e-1550-4804-b847-d4e3d67f7931 kind: Event lastTimestamp: "2026-05-20T14:03:18Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-83116-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-20T14:03:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:18Z" name: success-200-isvc-83116-predictor-59d5fb4b86-m27ms.18b14b01c57282d4 namespace: kserve-ci-e2e-test resourceVersion: "15712" uid: ed1e52a5-3576-4e3e-882b-91fd385a2b6a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:19Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-83116-predictor-59d5fb4b86-m27ms namespace: kserve-ci-e2e-test resourceVersion: "15682" uid: 5b59113e-1550-4804-b847-d4e3d67f7931 kind: Event lastTimestamp: "2026-05-20T14:03:19Z" message: Add eth0 [10.133.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T14:03:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T14:03:19Z" name: success-200-isvc-83116-predictor-59d5fb4b86-m27ms.18b14b01f72cba52 namespace: kserve-ci-e2e-test resourceVersion: "15739" uid: 3e1e9327-6265-4fa9-820e-88604a33257c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-83116-predictor-59d5fb4b86-m27ms namespace: kserve-ci-e2e-test resourceVersion: "15663" uid: 5b59113e-1550-4804-b847-d4e3d67f7931 kind: Event lastTimestamp: "2026-05-20T14:03:19Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1501" already present on machine metadata: creationTimestamp: "2026-05-20T14:03:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:19Z" name: success-200-isvc-83116-predictor-59d5fb4b86-m27ms.18b14b01f8d4c99a namespace: kserve-ci-e2e-test resourceVersion: "15741" uid: e9c75402-2cf3-41e6-81d3-b3bb1dc5f56f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-83116-predictor-59d5fb4b86-m27ms namespace: kserve-ci-e2e-test resourceVersion: "15663" uid: 5b59113e-1550-4804-b847-d4e3d67f7931 kind: Event lastTimestamp: "2026-05-20T14:03:19Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T14:03:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:19Z" name: success-200-isvc-83116-predictor-59d5fb4b86-m27ms.18b14b01fbc58ccc namespace: kserve-ci-e2e-test resourceVersion: "15743" uid: 317bf4ad-927b-48b5-ba59-e5b8f0145ca1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-83116-predictor-59d5fb4b86-m27ms namespace: kserve-ci-e2e-test resourceVersion: "15663" uid: 5b59113e-1550-4804-b847-d4e3d67f7931 kind: Event lastTimestamp: "2026-05-20T14:03:19Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T14:03:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:19Z" name: success-200-isvc-83116-predictor-59d5fb4b86-m27ms.18b14b01fc6ade36 namespace: kserve-ci-e2e-test resourceVersion: "15744" uid: 0a3318de-bebc-41fb-a754-0bad34cf9745 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-83116-predictor-59d5fb4b86-m27ms namespace: kserve-ci-e2e-test resourceVersion: "15663" uid: 5b59113e-1550-4804-b847-d4e3d67f7931 kind: Event lastTimestamp: "2026-05-20T14:03:19Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-20T14:03:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:19Z" name: success-200-isvc-83116-predictor-59d5fb4b86-m27ms.18b14b01fc76edfb namespace: kserve-ci-e2e-test resourceVersion: "15745" uid: 64196367-f179-46f6-9d62-96108ffadec4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-83116-predictor-59d5fb4b86-m27ms namespace: kserve-ci-e2e-test resourceVersion: "15663" uid: 5b59113e-1550-4804-b847-d4e3d67f7931 kind: Event lastTimestamp: "2026-05-20T14:03:19Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T14:03:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:19Z" name: success-200-isvc-83116-predictor-59d5fb4b86-m27ms.18b14b01fffc74bd namespace: kserve-ci-e2e-test resourceVersion: "15746" uid: 65b2842b-8ffb-4434-8380-4c812605f184 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-83116-predictor-59d5fb4b86-m27ms namespace: kserve-ci-e2e-test resourceVersion: "15663" uid: 5b59113e-1550-4804-b847-d4e3d67f7931 kind: Event lastTimestamp: "2026-05-20T14:03:19Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:03:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:19Z" name: success-200-isvc-83116-predictor-59d5fb4b86-m27ms.18b14b020062bc86 namespace: kserve-ci-e2e-test resourceVersion: "15747" uid: 41585fb6-6acf-434c-8bc1-3b5123539441 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-20T14:03:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-83116-predictor-59d5fb4b86-m27ms namespace: kserve-ci-e2e-test resourceVersion: "15663" uid: 5b59113e-1550-4804-b847-d4e3d67f7931 kind: Event lastTimestamp: "2026-05-20T14:03:57Z" message: 'Readiness probe failed: dial tcp 10.133.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T14:03:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:57Z" name: success-200-isvc-83116-predictor-59d5fb4b86-m27ms.18b14b025c4897b2 namespace: kserve-ci-e2e-test resourceVersion: "16287" uid: c474e36c-e8da-4e69-bede-5f1746e2756a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:12:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-83116-predictor-59d5fb4b86-m27ms namespace: kserve-ci-e2e-test resourceVersion: "15663" uid: 5b59113e-1550-4804-b847-d4e3d67f7931 kind: Event lastTimestamp: "2026-05-20T14:12:33Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T14:12:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:12:33Z" name: success-200-isvc-83116-predictor-59d5fb4b86-m27ms.18b14b82f7e37852 namespace: kserve-ci-e2e-test resourceVersion: "20026" uid: beb52a09-da17-4564-b930-776f9edcc59a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:12:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-83116-predictor-59d5fb4b86-m27ms namespace: kserve-ci-e2e-test resourceVersion: "15663" uid: 5b59113e-1550-4804-b847-d4e3d67f7931 kind: Event lastTimestamp: "2026-05-20T14:12:33Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:12:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:12:33Z" name: success-200-isvc-83116-predictor-59d5fb4b86-m27ms.18b14b82f7e3d117 namespace: kserve-ci-e2e-test resourceVersion: "20027" uid: ea2fcf49-78dd-4f1d-9347-45f42dbeffef reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:18Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-83116-predictor-59d5fb4b86 namespace: kserve-ci-e2e-test resourceVersion: "15648" uid: 8477af24-b126-459e-bf5f-36cfd61c115f kind: Event lastTimestamp: "2026-05-20T14:03:18Z" message: 'Created pod: success-200-isvc-83116-predictor-59d5fb4b86-m27ms' metadata: creationTimestamp: "2026-05-20T14:03:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:03:18Z" name: success-200-isvc-83116-predictor-59d5fb4b86.18b14b01b9979a9a namespace: kserve-ci-e2e-test resourceVersion: "15662" uid: 4e9852ef-c763-41c4-90c3-7c7a5b47d660 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:18Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-83116-predictor namespace: kserve-ci-e2e-test resourceVersion: "15641" uid: caa825be-7843-4e7a-980a-5436fcc09651 kind: Event lastTimestamp: "2026-05-20T14:03:18Z" message: Scaled up replica set success-200-isvc-83116-predictor-59d5fb4b86 from 0 to 1 metadata: creationTimestamp: "2026-05-20T14:03:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:03:18Z" name: success-200-isvc-83116-predictor.18b14b01b56bae46 namespace: kserve-ci-e2e-test resourceVersion: "15650" uid: 60382699-2f30-432f-a48d-572b3c9a7ed9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-20T14:03:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-83116-predictor namespace: kserve-ci-e2e-test resourceVersion: "15647" uid: 9a12edde-8230-4e15-bd69-3bf03b1d7ec6 kind: Event lastTimestamp: "2026-05-20T14:04:33Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:03:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:04:33Z" name: success-200-isvc-83116-predictor.18b14b05341dcc27 namespace: kserve-ci-e2e-test resourceVersion: "16637" uid: 3aa39d20-2763-4dc0-8a7c-0b111defb78c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-20T14:03:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-83116-predictor namespace: kserve-ci-e2e-test resourceVersion: "15647" uid: 9a12edde-8230-4e15-bd69-3bf03b1d7ec6 kind: Event lastTimestamp: "2026-05-20T14:04:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:03:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:04:33Z" name: success-200-isvc-83116-predictor.18b14b053477b89d namespace: kserve-ci-e2e-test resourceVersion: "16638" uid: 914731be-5e08-4bb5-98c2-5429b077db58 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-83116-predictor namespace: kserve-ci-e2e-test resourceVersion: "15875" uid: 9a12edde-8230-4e15-bd69-3bf03b1d7ec6 kind: Event lastTimestamp: "2026-05-20T14:03:48Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:03:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:03:48Z" name: success-200-isvc-83116-predictor.18b14b08b3997f0e namespace: kserve-ci-e2e-test resourceVersion: "16093" uid: 297385bc-c88a-44d0-94e8-98b7f1a2c214 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-83116-predictor namespace: kserve-ci-e2e-test resourceVersion: "15875" uid: 9a12edde-8230-4e15-bd69-3bf03b1d7ec6 kind: Event lastTimestamp: "2026-05-20T14:03:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:03:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:03:48Z" name: success-200-isvc-83116-predictor.18b14b08b44a05e8 namespace: kserve-ci-e2e-test resourceVersion: "16096" uid: 2fce17c1-7094-4527-8c8b-cdf30d703358 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-20T14:03:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-83116 namespace: kserve-ci-e2e-test resourceVersion: "15626" uid: f819fadf-3845-4577-8e4c-410fcfe10328 kind: Event lastTimestamp: "2026-05-20T14:04:07Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T14:03:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:04:07Z" name: success-200-isvc-83116.18b14b01b5374803 namespace: kserve-ci-e2e-test resourceVersion: "16390" uid: 368d15a2-1bb5-4e07-ac53-cd6f36633e30 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-83116 namespace: kserve-ci-e2e-test resourceVersion: "15626" uid: f819fadf-3845-4577-8e4c-410fcfe10328 kind: Event lastTimestamp: "2026-05-20T14:03:18Z" message: 'Failed to update status for InferenceService "success-200-isvc-83116": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-83116": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T14:03:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:03:18Z" name: success-200-isvc-83116.18b14b01b599c38b namespace: kserve-ci-e2e-test resourceVersion: "15653" uid: 83af8306-d57b-4925-814d-51cffa379301 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-83116 namespace: kserve-ci-e2e-test resourceVersion: "15626" uid: f819fadf-3845-4577-8e4c-410fcfe10328 kind: Event lastTimestamp: "2026-05-20T14:03:18Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-83116": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T14:03:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:03:18Z" name: success-200-isvc-83116.18b14b01b59a14a4 namespace: kserve-ci-e2e-test resourceVersion: "15654" uid: 727f4778-4d95-4fb3-8bfd-b772bb3126be reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:04:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-83116 namespace: kserve-ci-e2e-test resourceVersion: "16387" uid: f819fadf-3845-4577-8e4c-410fcfe10328 kind: Event lastTimestamp: "2026-05-20T14:04:07Z" message: InferenceService [success-200-isvc-83116] is Ready metadata: creationTimestamp: "2026-05-20T14:04:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:04:07Z" name: success-200-isvc-83116.18b14b0d17ed7c1d namespace: kserve-ci-e2e-test resourceVersion: "16388" uid: b1bffeb5-64c8-421b-a1b1-3f2b8c4abf61 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T14:22:58.802651Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a868a-predictor-7675b7748-5r6xk namespace: kserve-ci-e2e-test resourceVersion: "25278" uid: b7e32b31-f57b-4299-a06e-ae39800bbbdf kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-a868a-predictor-7675b7748-5r6xk to ip-10-0-137-148.ec2.internal metadata: creationTimestamp: "2026-05-20T14:22:58Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T14:22:58Z" name: success-200-isvc-a868a-predictor-7675b7748-5r6xk.18b14c148502b7ec namespace: kserve-ci-e2e-test resourceVersion: "25289" uid: 2cdea524-515d-4f19-89cb-b2e91d0502ce reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-747dc7bb84-jx2kx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:22:58Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a868a-predictor-7675b7748-5r6xk namespace: kserve-ci-e2e-test resourceVersion: "25282" uid: b7e32b31-f57b-4299-a06e-ae39800bbbdf kind: Event lastTimestamp: "2026-05-20T14:22:58Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-a868a-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-20T14:22:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:22:58Z" name: success-200-isvc-a868a-predictor-7675b7748-5r6xk.18b14c148d2f155e namespace: kserve-ci-e2e-test resourceVersion: "25328" uid: 51ffdabd-d901-42d0-8129-df2c7a4086a2 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:22:59Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a868a-predictor-7675b7748-5r6xk namespace: kserve-ci-e2e-test resourceVersion: "25306" uid: b7e32b31-f57b-4299-a06e-ae39800bbbdf kind: Event lastTimestamp: "2026-05-20T14:22:59Z" message: Add eth0 [10.133.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T14:22:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T14:22:59Z" name: success-200-isvc-a868a-predictor-7675b7748-5r6xk.18b14c14c1c90d30 namespace: kserve-ci-e2e-test resourceVersion: "25341" uid: f71d1801-9ccc-4c9c-9ac1-039550df7a9d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:22:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a868a-predictor-7675b7748-5r6xk namespace: kserve-ci-e2e-test resourceVersion: "25282" uid: b7e32b31-f57b-4299-a06e-ae39800bbbdf kind: Event lastTimestamp: "2026-05-20T14:22:59Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1501" already present on machine metadata: creationTimestamp: "2026-05-20T14:22:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:22:59Z" name: success-200-isvc-a868a-predictor-7675b7748-5r6xk.18b14c14c3de9a33 namespace: kserve-ci-e2e-test resourceVersion: "25344" uid: d3ed4d53-d094-46dc-ad52-cb5230765c42 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:22:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a868a-predictor-7675b7748-5r6xk namespace: kserve-ci-e2e-test resourceVersion: "25282" uid: b7e32b31-f57b-4299-a06e-ae39800bbbdf kind: Event lastTimestamp: "2026-05-20T14:22:59Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T14:22:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:22:59Z" name: success-200-isvc-a868a-predictor-7675b7748-5r6xk.18b14c14c6996578 namespace: kserve-ci-e2e-test resourceVersion: "25345" uid: afb6a888-8997-4804-afa6-886c1c42f10b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:22:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a868a-predictor-7675b7748-5r6xk namespace: kserve-ci-e2e-test resourceVersion: "25282" uid: b7e32b31-f57b-4299-a06e-ae39800bbbdf kind: Event lastTimestamp: "2026-05-20T14:22:59Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T14:22:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:22:59Z" name: success-200-isvc-a868a-predictor-7675b7748-5r6xk.18b14c14c703c409 namespace: kserve-ci-e2e-test resourceVersion: "25347" uid: c9add32e-ca9d-4b76-8190-6e7dcc543b87 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:22:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a868a-predictor-7675b7748-5r6xk namespace: kserve-ci-e2e-test resourceVersion: "25282" uid: b7e32b31-f57b-4299-a06e-ae39800bbbdf kind: Event lastTimestamp: "2026-05-20T14:22:59Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-20T14:22:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:22:59Z" name: success-200-isvc-a868a-predictor-7675b7748-5r6xk.18b14c14c70de4a3 namespace: kserve-ci-e2e-test resourceVersion: "25348" uid: 6b31fd99-3979-493b-836d-7921f12123cb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:22:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a868a-predictor-7675b7748-5r6xk namespace: kserve-ci-e2e-test resourceVersion: "25282" uid: b7e32b31-f57b-4299-a06e-ae39800bbbdf kind: Event lastTimestamp: "2026-05-20T14:22:59Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T14:22:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:22:59Z" name: success-200-isvc-a868a-predictor-7675b7748-5r6xk.18b14c14ca151c66 namespace: kserve-ci-e2e-test resourceVersion: "25351" uid: c1f32335-52b6-4667-bdd5-00bf1564b8c0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:22:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a868a-predictor-7675b7748-5r6xk namespace: kserve-ci-e2e-test resourceVersion: "25282" uid: b7e32b31-f57b-4299-a06e-ae39800bbbdf kind: Event lastTimestamp: "2026-05-20T14:22:59Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:22:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:22:59Z" name: success-200-isvc-a868a-predictor-7675b7748-5r6xk.18b14c14ca8204a9 namespace: kserve-ci-e2e-test resourceVersion: "25352" uid: d54edbff-3c87-4684-9845-5129c3b44e1e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-20T14:23:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a868a-predictor-7675b7748-5r6xk namespace: kserve-ci-e2e-test resourceVersion: "25282" uid: b7e32b31-f57b-4299-a06e-ae39800bbbdf kind: Event lastTimestamp: "2026-05-20T14:23:36Z" message: 'Readiness probe failed: dial tcp 10.133.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T14:23:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:23:36Z" name: success-200-isvc-a868a-predictor-7675b7748-5r6xk.18b14c14f31fa610 namespace: kserve-ci-e2e-test resourceVersion: "25801" uid: 3031c1c2-c5e6-44af-9f61-2d7f38662e50 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:24:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a868a-predictor-7675b7748-5r6xk namespace: kserve-ci-e2e-test resourceVersion: "25282" uid: b7e32b31-f57b-4299-a06e-ae39800bbbdf kind: Event lastTimestamp: "2026-05-20T14:24:09Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T14:24:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:24:09Z" name: success-200-isvc-a868a-predictor-7675b7748-5r6xk.18b14c24e1e523e8 namespace: kserve-ci-e2e-test resourceVersion: "26471" uid: eb76a0cd-fa84-4cd5-a12d-0cd969e6804c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:24:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a868a-predictor-7675b7748-5r6xk namespace: kserve-ci-e2e-test resourceVersion: "25282" uid: b7e32b31-f57b-4299-a06e-ae39800bbbdf kind: Event lastTimestamp: "2026-05-20T14:24:09Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:24:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:24:09Z" name: success-200-isvc-a868a-predictor-7675b7748-5r6xk.18b14c24e1e6398b namespace: kserve-ci-e2e-test resourceVersion: "26473" uid: cc2e604f-e88a-471d-86b1-936e6f202556 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:24:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a868a-predictor-7675b7748-5r6xk namespace: kserve-ci-e2e-test resourceVersion: "25282" uid: b7e32b31-f57b-4299-a06e-ae39800bbbdf kind: Event lastTimestamp: "2026-05-20T14:24:11Z" message: 'Readiness probe failed: Get "https://10.133.0.28:8643/healthz": dial tcp 10.133.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-20T14:24:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:24:11Z" name: success-200-isvc-a868a-predictor-7675b7748-5r6xk.18b14c257b3bb23b namespace: kserve-ci-e2e-test resourceVersion: "26574" uid: 04309143-9f77-48eb-a0a0-6441f17a16a7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:22:58Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-a868a-predictor-7675b7748 namespace: kserve-ci-e2e-test resourceVersion: "25263" uid: 5934406d-96f6-44c2-9215-6f9fb19297e6 kind: Event lastTimestamp: "2026-05-20T14:22:58Z" message: 'Created pod: success-200-isvc-a868a-predictor-7675b7748-5r6xk' metadata: creationTimestamp: "2026-05-20T14:22:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:22:58Z" name: success-200-isvc-a868a-predictor-7675b7748.18b14c1484824ccf namespace: kserve-ci-e2e-test resourceVersion: "25280" uid: 9540fc88-d2d2-4432-9e37-8849f3b736ec reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:22:58Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-a868a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25256" uid: 733e592f-ab3a-475d-8377-4d2a1d3695fa kind: Event lastTimestamp: "2026-05-20T14:22:58Z" message: Scaled up replica set success-200-isvc-a868a-predictor-7675b7748 from 0 to 1 metadata: creationTimestamp: "2026-05-20T14:22:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:22:58Z" name: success-200-isvc-a868a-predictor.18b14c1480920b44 namespace: kserve-ci-e2e-test resourceVersion: "25266" uid: 8eb2a75d-0c6c-4017-94b5-3ab7a6b44a41 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T14:23:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a868a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25259" uid: 49938fe4-f944-407a-a396-456321a30e4f kind: Event lastTimestamp: "2026-05-20T14:23:28Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:23:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:23:28Z" name: success-200-isvc-a868a-predictor.18b14c17ffb58d07 namespace: kserve-ci-e2e-test resourceVersion: "25687" uid: c82cc3ee-44cf-4f4d-aacc-aacc72afe4c0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T14:23:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a868a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25259" uid: 49938fe4-f944-407a-a396-456321a30e4f kind: Event lastTimestamp: "2026-05-20T14:23:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:23:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:23:28Z" name: success-200-isvc-a868a-predictor.18b14c18002d5935 namespace: kserve-ci-e2e-test resourceVersion: "25691" uid: 4df52c8d-d958-4fe9-b450-e3d391e20435 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T14:23:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a868a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25477" uid: 49938fe4-f944-407a-a396-456321a30e4f kind: Event lastTimestamp: "2026-05-20T14:23:58Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:23:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:23:58Z" name: success-200-isvc-a868a-predictor.18b14c1efe858480 namespace: kserve-ci-e2e-test resourceVersion: "26298" uid: cf9c9cc8-3ae0-41c5-8b3c-9a0a0c2ebb4c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T14:23:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a868a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25477" uid: 49938fe4-f944-407a-a396-456321a30e4f kind: Event lastTimestamp: "2026-05-20T14:23:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:23:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:23:58Z" name: success-200-isvc-a868a-predictor.18b14c1eff01b0b7 namespace: kserve-ci-e2e-test resourceVersion: "26302" uid: e5614fad-bc0c-4d3a-a9fb-eaecb59a60ec reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-20T14:22:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a868a namespace: kserve-ci-e2e-test resourceVersion: "25244" uid: 7212230c-5552-4265-b02f-3dd359124717 kind: Event lastTimestamp: "2026-05-20T14:23:46Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T14:22:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:23:46Z" name: success-200-isvc-a868a.18b14c148061bbd1 namespace: kserve-ci-e2e-test resourceVersion: "26186" uid: bcdb3159-30a7-47a9-91b1-2441719f59cc reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:22:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a868a namespace: kserve-ci-e2e-test resourceVersion: "25244" uid: 7212230c-5552-4265-b02f-3dd359124717 kind: Event lastTimestamp: "2026-05-20T14:22:58Z" message: 'Failed to update status for InferenceService "success-200-isvc-a868a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-a868a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T14:22:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:22:58Z" name: success-200-isvc-a868a.18b14c14813262a3 namespace: kserve-ci-e2e-test resourceVersion: "25269" uid: 06eb6aaf-749f-4e45-9deb-f085329e65a7 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:22:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a868a namespace: kserve-ci-e2e-test resourceVersion: "25244" uid: 7212230c-5552-4265-b02f-3dd359124717 kind: Event lastTimestamp: "2026-05-20T14:22:58Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-a868a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T14:22:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:22:58Z" name: success-200-isvc-a868a.18b14c148132a88a namespace: kserve-ci-e2e-test resourceVersion: "25272" uid: 14dfd9e2-277d-4a6e-8a2d-ca99a8f58a9c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:23:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a868a namespace: kserve-ci-e2e-test resourceVersion: "26183" uid: 7212230c-5552-4265-b02f-3dd359124717 kind: Event lastTimestamp: "2026-05-20T14:23:46Z" message: InferenceService [success-200-isvc-a868a] is Ready metadata: creationTimestamp: "2026-05-20T14:23:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:23:46Z" name: success-200-isvc-a868a.18b14c1faf7e555b namespace: kserve-ci-e2e-test resourceVersion: "26184" uid: bf395c12-6b2b-4dfc-860a-ecd05e8096bc reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T14:03:48.583531Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-cdd1d-predictor-685c7f8cd8-5jgsv namespace: kserve-ci-e2e-test resourceVersion: "16099" uid: c89fd641-6384-45fc-9a9b-eb65d310c2e3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-cdd1d-predictor-685c7f8cd8-5jgsv to ip-10-0-137-148.ec2.internal metadata: creationTimestamp: "2026-05-20T14:03:48Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T14:03:48Z" name: success-200-isvc-cdd1d-predictor-685c7f8cd8-5jgsv.18b14b08b69bc22d namespace: kserve-ci-e2e-test resourceVersion: "16107" uid: 3012c840-77f0-4e3c-a58f-bb814c82856c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-747dc7bb84-jx2kx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:48Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-cdd1d-predictor-685c7f8cd8-5jgsv namespace: kserve-ci-e2e-test resourceVersion: "16103" uid: c89fd641-6384-45fc-9a9b-eb65d310c2e3 kind: Event lastTimestamp: "2026-05-20T14:03:48Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-cdd1d-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-20T14:03:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:48Z" name: success-200-isvc-cdd1d-predictor-685c7f8cd8-5jgsv.18b14b08c5ed900f namespace: kserve-ci-e2e-test resourceVersion: "16175" uid: 286a0645-b8a6-4c75-8d79-021ee03b1e0c reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:49Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-cdd1d-predictor-685c7f8cd8-5jgsv namespace: kserve-ci-e2e-test resourceVersion: "16116" uid: c89fd641-6384-45fc-9a9b-eb65d310c2e3 kind: Event lastTimestamp: "2026-05-20T14:03:49Z" message: Add eth0 [10.133.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T14:03:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T14:03:49Z" name: success-200-isvc-cdd1d-predictor-685c7f8cd8-5jgsv.18b14b08f40ca04d namespace: kserve-ci-e2e-test resourceVersion: "16193" uid: 1a13902d-abca-4803-acb6-65c2f0d7c963 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cdd1d-predictor-685c7f8cd8-5jgsv namespace: kserve-ci-e2e-test resourceVersion: "16103" uid: c89fd641-6384-45fc-9a9b-eb65d310c2e3 kind: Event lastTimestamp: "2026-05-20T14:03:49Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1501" already present on machine metadata: creationTimestamp: "2026-05-20T14:03:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:49Z" name: success-200-isvc-cdd1d-predictor-685c7f8cd8-5jgsv.18b14b08f5917037 namespace: kserve-ci-e2e-test resourceVersion: "16195" uid: 34a61f4c-ceb9-4508-b87b-fcc4231ffb8f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cdd1d-predictor-685c7f8cd8-5jgsv namespace: kserve-ci-e2e-test resourceVersion: "16103" uid: c89fd641-6384-45fc-9a9b-eb65d310c2e3 kind: Event lastTimestamp: "2026-05-20T14:03:49Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T14:03:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:49Z" name: success-200-isvc-cdd1d-predictor-685c7f8cd8-5jgsv.18b14b08f88ea2b1 namespace: kserve-ci-e2e-test resourceVersion: "16196" uid: a8cecf22-e921-453b-9cdc-e2ebdf236c6d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cdd1d-predictor-685c7f8cd8-5jgsv namespace: kserve-ci-e2e-test resourceVersion: "16103" uid: c89fd641-6384-45fc-9a9b-eb65d310c2e3 kind: Event lastTimestamp: "2026-05-20T14:03:49Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T14:03:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:49Z" name: success-200-isvc-cdd1d-predictor-685c7f8cd8-5jgsv.18b14b08f8f032bc namespace: kserve-ci-e2e-test resourceVersion: "16197" uid: 7d4e26b1-8e4b-4bec-b41f-55b9be199008 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-cdd1d-predictor-685c7f8cd8-5jgsv namespace: kserve-ci-e2e-test resourceVersion: "16103" uid: c89fd641-6384-45fc-9a9b-eb65d310c2e3 kind: Event lastTimestamp: "2026-05-20T14:03:49Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-20T14:03:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:49Z" name: success-200-isvc-cdd1d-predictor-685c7f8cd8-5jgsv.18b14b08f8fc80be namespace: kserve-ci-e2e-test resourceVersion: "16198" uid: 088286f9-67e9-45ed-b802-0ca7876cc7fd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-cdd1d-predictor-685c7f8cd8-5jgsv namespace: kserve-ci-e2e-test resourceVersion: "16103" uid: c89fd641-6384-45fc-9a9b-eb65d310c2e3 kind: Event lastTimestamp: "2026-05-20T14:03:49Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T14:03:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:49Z" name: success-200-isvc-cdd1d-predictor-685c7f8cd8-5jgsv.18b14b08fb8c4807 namespace: kserve-ci-e2e-test resourceVersion: "16201" uid: 90753b6a-f7b0-488d-b6da-fd00ec4f3a15 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-cdd1d-predictor-685c7f8cd8-5jgsv namespace: kserve-ci-e2e-test resourceVersion: "16103" uid: c89fd641-6384-45fc-9a9b-eb65d310c2e3 kind: Event lastTimestamp: "2026-05-20T14:03:49Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:03:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:49Z" name: success-200-isvc-cdd1d-predictor-685c7f8cd8-5jgsv.18b14b08fc6a0620 namespace: kserve-ci-e2e-test resourceVersion: "16203" uid: 438c9f17-16e1-4ddb-95d2-0387667b1694 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-20T14:03:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cdd1d-predictor-685c7f8cd8-5jgsv namespace: kserve-ci-e2e-test resourceVersion: "16103" uid: c89fd641-6384-45fc-9a9b-eb65d310c2e3 kind: Event lastTimestamp: "2026-05-20T14:04:27Z" message: 'Readiness probe failed: dial tcp 10.133.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T14:03:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:04:27Z" name: success-200-isvc-cdd1d-predictor-685c7f8cd8-5jgsv.18b14b095e277fe7 namespace: kserve-ci-e2e-test resourceVersion: "16588" uid: 8fa12ce5-62ed-40ed-832e-1e7e54cd3bbb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cdd1d-predictor-685c7f8cd8-5jgsv namespace: kserve-ci-e2e-test resourceVersion: "16103" uid: c89fd641-6384-45fc-9a9b-eb65d310c2e3 kind: Event lastTimestamp: "2026-05-20T14:13:03Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T14:13:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:13:03Z" name: success-200-isvc-cdd1d-predictor-685c7f8cd8-5jgsv.18b14b89e59bed0e namespace: kserve-ci-e2e-test resourceVersion: "20386" uid: 3adbf6de-1088-42d6-bc8d-1d9c8b37f487 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-cdd1d-predictor-685c7f8cd8-5jgsv namespace: kserve-ci-e2e-test resourceVersion: "16103" uid: c89fd641-6384-45fc-9a9b-eb65d310c2e3 kind: Event lastTimestamp: "2026-05-20T14:13:03Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:13:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:13:03Z" name: success-200-isvc-cdd1d-predictor-685c7f8cd8-5jgsv.18b14b89e59c6cbe namespace: kserve-ci-e2e-test resourceVersion: "20390" uid: 1c78ddce-c2aa-49dc-87d7-493eb5f20c94 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:48Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-cdd1d-predictor-685c7f8cd8 namespace: kserve-ci-e2e-test resourceVersion: "16081" uid: 1faaf671-17e8-44ee-a469-3309ff5ef19a kind: Event lastTimestamp: "2026-05-20T14:03:48Z" message: 'Created pod: success-200-isvc-cdd1d-predictor-685c7f8cd8-5jgsv' metadata: creationTimestamp: "2026-05-20T14:03:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:03:48Z" name: success-200-isvc-cdd1d-predictor-685c7f8cd8.18b14b08b5ec8a65 namespace: kserve-ci-e2e-test resourceVersion: "16102" uid: b2e53f91-575d-4eb8-a03b-7f290f2c89bc reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:48Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-cdd1d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16073" uid: 100514ca-446d-4164-b94d-faae014c3fcc kind: Event lastTimestamp: "2026-05-20T14:03:48Z" message: Scaled up replica set success-200-isvc-cdd1d-predictor-685c7f8cd8 from 0 to 1 metadata: creationTimestamp: "2026-05-20T14:03:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:03:48Z" name: success-200-isvc-cdd1d-predictor.18b14b08b321d05d namespace: kserve-ci-e2e-test resourceVersion: "16086" uid: 2d990d50-d262-4e91-849b-84947ea4b48e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-20T14:04:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-cdd1d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16078" uid: 0b387da0-367e-41d5-b88b-95d78b92a49a kind: Event lastTimestamp: "2026-05-20T14:05:03Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:04:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:05:03Z" name: success-200-isvc-cdd1d-predictor.18b14b0c31b98c57 namespace: kserve-ci-e2e-test resourceVersion: "16921" uid: 9d4bfa38-a821-4365-8aee-32361edc9aa8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-20T14:04:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-cdd1d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16078" uid: 0b387da0-367e-41d5-b88b-95d78b92a49a kind: Event lastTimestamp: "2026-05-20T14:05:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:04:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:05:03Z" name: success-200-isvc-cdd1d-predictor.18b14b0c32381c07 namespace: kserve-ci-e2e-test resourceVersion: "16922" uid: 17813d16-2292-4ab7-88e6-79803a75d6f5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:04:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-cdd1d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16331" uid: 0b387da0-367e-41d5-b88b-95d78b92a49a kind: Event lastTimestamp: "2026-05-20T14:04:18Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:04:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:04:18Z" name: success-200-isvc-cdd1d-predictor.18b14b0fb1807387 namespace: kserve-ci-e2e-test resourceVersion: "16489" uid: 68f14690-0950-43ad-aa53-208cefa30ec0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:04:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-cdd1d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16331" uid: 0b387da0-367e-41d5-b88b-95d78b92a49a kind: Event lastTimestamp: "2026-05-20T14:04:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:04:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:04:18Z" name: success-200-isvc-cdd1d-predictor.18b14b0fb214c92b namespace: kserve-ci-e2e-test resourceVersion: "16493" uid: 46bf155a-3052-499d-b20e-ae7cdec80d8d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-20T14:03:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-cdd1d namespace: kserve-ci-e2e-test resourceVersion: "16063" uid: d655ca8e-7d5c-48ab-a7cd-3a022473717c kind: Event lastTimestamp: "2026-05-20T14:04:37Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T14:03:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:04:37Z" name: success-200-isvc-cdd1d.18b14b08b2e099e4 namespace: kserve-ci-e2e-test resourceVersion: "16690" uid: f1091219-5141-4b50-989d-bb1643cabf6f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-cdd1d namespace: kserve-ci-e2e-test resourceVersion: "16063" uid: d655ca8e-7d5c-48ab-a7cd-3a022473717c kind: Event lastTimestamp: "2026-05-20T14:03:48Z" message: 'Failed to update status for InferenceService "success-200-isvc-cdd1d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-cdd1d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T14:03:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:03:48Z" name: success-200-isvc-cdd1d.18b14b08b39be671 namespace: kserve-ci-e2e-test resourceVersion: "16091" uid: 961f2d32-4880-4b92-8cb1-99681721573d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-cdd1d namespace: kserve-ci-e2e-test resourceVersion: "16063" uid: d655ca8e-7d5c-48ab-a7cd-3a022473717c kind: Event lastTimestamp: "2026-05-20T14:03:48Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-cdd1d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T14:03:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:03:48Z" name: success-200-isvc-cdd1d.18b14b08b39c5048 namespace: kserve-ci-e2e-test resourceVersion: "16095" uid: 75853913-4f7e-424c-9314-f8f772767a55 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:04:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-cdd1d namespace: kserve-ci-e2e-test resourceVersion: "16688" uid: d655ca8e-7d5c-48ab-a7cd-3a022473717c kind: Event lastTimestamp: "2026-05-20T14:04:37Z" message: InferenceService [success-200-isvc-cdd1d] is Ready metadata: creationTimestamp: "2026-05-20T14:04:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:04:37Z" name: success-200-isvc-cdd1d.18b14b141949ec07 namespace: kserve-ci-e2e-test resourceVersion: "16689" uid: 987fc246-fee3-4ca0-b7ef-7e13a7651875 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T14:13:43.929804Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ecafe-predictor-785fb74b88-nx9dq namespace: kserve-ci-e2e-test resourceVersion: "20912" uid: 6a1303af-b906-402f-9ba8-44501594a4b4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-ecafe-predictor-785fb74b88-nx9dq to ip-10-0-137-148.ec2.internal metadata: creationTimestamp: "2026-05-20T14:13:43Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T14:13:43Z" name: success-200-isvc-ecafe-predictor-785fb74b88-nx9dq.18b14b935402f91e namespace: kserve-ci-e2e-test resourceVersion: "20919" uid: 70ff2131-4616-4331-94d6-2d2bfaa3d9f2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-747dc7bb84-jx2kx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:44Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ecafe-predictor-785fb74b88-nx9dq namespace: kserve-ci-e2e-test resourceVersion: "20943" uid: 6a1303af-b906-402f-9ba8-44501594a4b4 kind: Event lastTimestamp: "2026-05-20T14:13:44Z" message: Add eth0 [10.133.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T14:13:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T14:13:44Z" name: success-200-isvc-ecafe-predictor-785fb74b88-nx9dq.18b14b936d1c759f namespace: kserve-ci-e2e-test resourceVersion: "20974" uid: ff3c5931-4db3-4f19-9b1c-22a870dbbbd1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ecafe-predictor-785fb74b88-nx9dq namespace: kserve-ci-e2e-test resourceVersion: "20916" uid: 6a1303af-b906-402f-9ba8-44501594a4b4 kind: Event lastTimestamp: "2026-05-20T14:13:44Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1501" already present on machine metadata: creationTimestamp: "2026-05-20T14:13:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:13:44Z" name: success-200-isvc-ecafe-predictor-785fb74b88-nx9dq.18b14b936e9175c7 namespace: kserve-ci-e2e-test resourceVersion: "20977" uid: f61d7d01-3dd1-4319-82cf-174588ae0997 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ecafe-predictor-785fb74b88-nx9dq namespace: kserve-ci-e2e-test resourceVersion: "20916" uid: 6a1303af-b906-402f-9ba8-44501594a4b4 kind: Event lastTimestamp: "2026-05-20T14:13:44Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T14:13:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:13:44Z" name: success-200-isvc-ecafe-predictor-785fb74b88-nx9dq.18b14b9371ae654f namespace: kserve-ci-e2e-test resourceVersion: "20978" uid: 2c99be74-442e-4f32-a447-828192d3b805 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ecafe-predictor-785fb74b88-nx9dq namespace: kserve-ci-e2e-test resourceVersion: "20916" uid: 6a1303af-b906-402f-9ba8-44501594a4b4 kind: Event lastTimestamp: "2026-05-20T14:13:44Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T14:13:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:13:44Z" name: success-200-isvc-ecafe-predictor-785fb74b88-nx9dq.18b14b93721d36e9 namespace: kserve-ci-e2e-test resourceVersion: "20979" uid: 82c2e90e-f1cd-4b2e-9782-610928039480 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ecafe-predictor-785fb74b88-nx9dq namespace: kserve-ci-e2e-test resourceVersion: "20916" uid: 6a1303af-b906-402f-9ba8-44501594a4b4 kind: Event lastTimestamp: "2026-05-20T14:13:44Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-20T14:13:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:13:44Z" name: success-200-isvc-ecafe-predictor-785fb74b88-nx9dq.18b14b93722846f1 namespace: kserve-ci-e2e-test resourceVersion: "20980" uid: 0a20bab5-09cc-4d03-91e3-e2a1a3d086dd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ecafe-predictor-785fb74b88-nx9dq namespace: kserve-ci-e2e-test resourceVersion: "20916" uid: 6a1303af-b906-402f-9ba8-44501594a4b4 kind: Event lastTimestamp: "2026-05-20T14:13:44Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T14:13:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:13:44Z" name: success-200-isvc-ecafe-predictor-785fb74b88-nx9dq.18b14b9374db7ff5 namespace: kserve-ci-e2e-test resourceVersion: "20981" uid: 76af9b2f-c832-4498-b45d-879219d5272c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ecafe-predictor-785fb74b88-nx9dq namespace: kserve-ci-e2e-test resourceVersion: "20916" uid: 6a1303af-b906-402f-9ba8-44501594a4b4 kind: Event lastTimestamp: "2026-05-20T14:13:44Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:13:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:13:44Z" name: success-200-isvc-ecafe-predictor-785fb74b88-nx9dq.18b14b9375466117 namespace: kserve-ci-e2e-test resourceVersion: "20982" uid: 66b0e5f1-8323-460f-9422-ebd8e03078d5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-05-20T14:13:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ecafe-predictor-785fb74b88-nx9dq namespace: kserve-ci-e2e-test resourceVersion: "20916" uid: 6a1303af-b906-402f-9ba8-44501594a4b4 kind: Event lastTimestamp: "2026-05-20T14:23:01Z" message: 'Readiness probe failed: dial tcp 10.133.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T14:13:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:23:01Z" name: success-200-isvc-ecafe-predictor-785fb74b88-nx9dq.18b14b9399c5caf4 namespace: kserve-ci-e2e-test resourceVersion: "25377" uid: 5e7f266a-684d-4402-9d80-7314f4f852d2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:22:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ecafe-predictor-785fb74b88-nx9dq namespace: kserve-ci-e2e-test resourceVersion: "20916" uid: 6a1303af-b906-402f-9ba8-44501594a4b4 kind: Event lastTimestamp: "2026-05-20T14:22:58Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T14:22:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:22:58Z" name: success-200-isvc-ecafe-predictor-785fb74b88-nx9dq.18b14c148212981f namespace: kserve-ci-e2e-test resourceVersion: "25274" uid: 4d7560cd-8bda-4f0c-b55d-ca728c5c4897 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:22:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ecafe-predictor-785fb74b88-nx9dq namespace: kserve-ci-e2e-test resourceVersion: "20916" uid: 6a1303af-b906-402f-9ba8-44501594a4b4 kind: Event lastTimestamp: "2026-05-20T14:22:58Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:22:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:22:58Z" name: success-200-isvc-ecafe-predictor-785fb74b88-nx9dq.18b14c1482131a9f namespace: kserve-ci-e2e-test resourceVersion: "25275" uid: 97f1fe20-f36d-405d-8bfa-139a2b4c7a11 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:23:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ecafe-predictor-785fb74b88-nx9dq namespace: kserve-ci-e2e-test resourceVersion: "20916" uid: 6a1303af-b906-402f-9ba8-44501594a4b4 kind: Event lastTimestamp: "2026-05-20T14:23:01Z" message: 'Readiness probe failed: Get "https://10.133.0.26:8643/healthz": dial tcp 10.133.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-20T14:23:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:23:01Z" name: success-200-isvc-ecafe-predictor-785fb74b88-nx9dq.18b14c150e1af6a1 namespace: kserve-ci-e2e-test resourceVersion: "25375" uid: 449e2ca7-728d-45ac-93e4-68bb5d1191d3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-148.ec2.internal source: component: kubelet host: ip-10-0-137-148.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:43Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-ecafe-predictor-785fb74b88 namespace: kserve-ci-e2e-test resourceVersion: "20900" uid: 56871e1f-b614-4db8-a69c-99be25fa4861 kind: Event lastTimestamp: "2026-05-20T14:13:43Z" message: 'Created pod: success-200-isvc-ecafe-predictor-785fb74b88-nx9dq' metadata: creationTimestamp: "2026-05-20T14:13:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:13:43Z" name: success-200-isvc-ecafe-predictor-785fb74b88.18b14b93538419b6 namespace: kserve-ci-e2e-test resourceVersion: "20917" uid: 5f01ccbb-6764-488c-8ff7-ddede6607ead reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:43Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-ecafe-predictor namespace: kserve-ci-e2e-test resourceVersion: "20894" uid: 33719a45-e5a4-46e0-9975-06ff47bce495 kind: Event lastTimestamp: "2026-05-20T14:13:43Z" message: Scaled up replica set success-200-isvc-ecafe-predictor-785fb74b88 from 0 to 1 metadata: creationTimestamp: "2026-05-20T14:13:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:13:43Z" name: success-200-isvc-ecafe-predictor.18b14b93510b26bb namespace: kserve-ci-e2e-test resourceVersion: "20906" uid: 4ce39c11-02a5-4cd2-90fb-424987481d06 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T14:13:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ecafe-predictor namespace: kserve-ci-e2e-test resourceVersion: "20897" uid: cf3484b4-78f7-4f60-8093-6c074d6d3fee kind: Event lastTimestamp: "2026-05-20T14:14:13Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:13:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:14:13Z" name: success-200-isvc-ecafe-predictor.18b14b96cfe41324 namespace: kserve-ci-e2e-test resourceVersion: "21406" uid: 3c2b4464-f4f4-489c-a227-eaa97c0e7850 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T14:13:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ecafe-predictor namespace: kserve-ci-e2e-test resourceVersion: "20897" uid: cf3484b4-78f7-4f60-8093-6c074d6d3fee kind: Event lastTimestamp: "2026-05-20T14:14:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:13:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:14:13Z" name: success-200-isvc-ecafe-predictor.18b14b96d0b38a27 namespace: kserve-ci-e2e-test resourceVersion: "21407" uid: 556d102e-f2cd-4dfd-bc22-fa01d0ea7c62 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T14:14:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ecafe-predictor namespace: kserve-ci-e2e-test resourceVersion: "21135" uid: cf3484b4-78f7-4f60-8093-6c074d6d3fee kind: Event lastTimestamp: "2026-05-20T14:14:58Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:14:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:14:58Z" name: success-200-isvc-ecafe-predictor.18b14b9dce4beccb namespace: kserve-ci-e2e-test resourceVersion: "21867" uid: cb17ad85-e6fe-4df2-80f0-3f61c5fc9898 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T14:14:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ecafe-predictor namespace: kserve-ci-e2e-test resourceVersion: "21135" uid: cf3484b4-78f7-4f60-8093-6c074d6d3fee kind: Event lastTimestamp: "2026-05-20T14:14:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:14:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:14:58Z" name: success-200-isvc-ecafe-predictor.18b14b9dcebf2aa8 namespace: kserve-ci-e2e-test resourceVersion: "21868" uid: e82c8173-c4f5-4cd4-beac-54559fdbbc80 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-20T14:13:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ecafe namespace: kserve-ci-e2e-test resourceVersion: "20881" uid: fc5c3bc3-e744-4a5c-9556-9e074f12e994 kind: Event lastTimestamp: "2026-05-20T14:14:31Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T14:13:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:14:31Z" name: success-200-isvc-ecafe.18b14b9350acc614 namespace: kserve-ci-e2e-test resourceVersion: "21598" uid: b3ecf2c4-d6c2-48b9-a793-3497fb90bc1a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ecafe namespace: kserve-ci-e2e-test resourceVersion: "20881" uid: fc5c3bc3-e744-4a5c-9556-9e074f12e994 kind: Event lastTimestamp: "2026-05-20T14:13:43Z" message: 'Failed to update status for InferenceService "success-200-isvc-ecafe": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-ecafe": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T14:13:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:13:43Z" name: success-200-isvc-ecafe.18b14b93514cb6e8 namespace: kserve-ci-e2e-test resourceVersion: "20908" uid: 8570db8f-e207-4bde-bce6-894bed9f8fed reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:13:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ecafe namespace: kserve-ci-e2e-test resourceVersion: "20881" uid: fc5c3bc3-e744-4a5c-9556-9e074f12e994 kind: Event lastTimestamp: "2026-05-20T14:13:43Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-ecafe": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T14:13:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:13:43Z" name: success-200-isvc-ecafe.18b14b93514d0f07 namespace: kserve-ci-e2e-test resourceVersion: "20910" uid: 8e9ea89c-8aed-492f-a4c5-88d354a58642 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:14:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ecafe namespace: kserve-ci-e2e-test resourceVersion: "21596" uid: fc5c3bc3-e744-4a5c-9556-9e074f12e994 kind: Event lastTimestamp: "2026-05-20T14:14:31Z" message: InferenceService [success-200-isvc-ecafe] is Ready metadata: creationTimestamp: "2026-05-20T14:14:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:14:31Z" name: success-200-isvc-ecafe.18b14b9e55b08326 namespace: kserve-ci-e2e-test resourceVersion: "21597" uid: 01156cd9-5b7c-45b9-a8c3-c1ececc1caf4 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T14:24:28.658882Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-3e303-66c47b978f-nfvht namespace: kserve-ci-e2e-test resourceVersion: "26746" uid: 14cab6c0-a554-416d-ae2e-76b2c089e696 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-3e303-66c47b978f-nfvht to ip-10-0-136-75.ec2.internal metadata: creationTimestamp: "2026-05-20T14:24:28Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T14:24:28Z" name: switch-graph-3e303-66c47b978f-nfvht.18b14c2970dbf4aa namespace: kserve-ci-e2e-test resourceVersion: "26753" uid: 26b99211-62d6-4fe3-9a13-ee60b45da172 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-747dc7bb84-jx2kx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:24:28Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-3e303-66c47b978f-nfvht namespace: kserve-ci-e2e-test resourceVersion: "26748" uid: 14cab6c0-a554-416d-ae2e-76b2c089e696 kind: Event lastTimestamp: "2026-05-20T14:24:28Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-3e303-serving-cert" not found' metadata: creationTimestamp: "2026-05-20T14:24:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:24:28Z" name: switch-graph-3e303-66c47b978f-nfvht.18b14c297c7ef5ab namespace: kserve-ci-e2e-test resourceVersion: "26768" uid: 585c74ec-122a-4ddc-9bf6-b799d21f60d6 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:24:29Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-3e303-66c47b978f-nfvht namespace: kserve-ci-e2e-test resourceVersion: "26758" uid: 14cab6c0-a554-416d-ae2e-76b2c089e696 kind: Event lastTimestamp: "2026-05-20T14:24:29Z" message: Add eth0 [10.134.0.50/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T14:24:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T14:24:29Z" name: switch-graph-3e303-66c47b978f-nfvht.18b14c29aead88c8 namespace: kserve-ci-e2e-test resourceVersion: "26779" uid: 4db332ca-7056-4549-ae4c-0709969373ac reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:24:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-3e303} kind: Pod name: switch-graph-3e303-66c47b978f-nfvht namespace: kserve-ci-e2e-test resourceVersion: "26748" uid: 14cab6c0-a554-416d-ae2e-76b2c089e696 kind: Event lastTimestamp: "2026-05-20T14:24:29Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:04926a9d7185ff179745ae32ed77ae962904508dbcb7541e9bf600392af6f503" already present on machine metadata: creationTimestamp: "2026-05-20T14:24:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:24:29Z" name: switch-graph-3e303-66c47b978f-nfvht.18b14c29b022fa0d namespace: kserve-ci-e2e-test resourceVersion: "26782" uid: 279a165e-cbfb-4ae1-b98d-7fa6c3a6c95a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:24:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-3e303} kind: Pod name: switch-graph-3e303-66c47b978f-nfvht namespace: kserve-ci-e2e-test resourceVersion: "26748" uid: 14cab6c0-a554-416d-ae2e-76b2c089e696 kind: Event lastTimestamp: "2026-05-20T14:24:29Z" message: 'Created container: switch-graph-3e303' metadata: creationTimestamp: "2026-05-20T14:24:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:24:29Z" name: switch-graph-3e303-66c47b978f-nfvht.18b14c29b315e24e namespace: kserve-ci-e2e-test resourceVersion: "26783" uid: 7e28f208-7375-4991-bdbf-a005c83042f4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:24:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-3e303} kind: Pod name: switch-graph-3e303-66c47b978f-nfvht namespace: kserve-ci-e2e-test resourceVersion: "26748" uid: 14cab6c0-a554-416d-ae2e-76b2c089e696 kind: Event lastTimestamp: "2026-05-20T14:24:29Z" message: Started container switch-graph-3e303 metadata: creationTimestamp: "2026-05-20T14:24:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:24:29Z" name: switch-graph-3e303-66c47b978f-nfvht.18b14c29b3943f25 namespace: kserve-ci-e2e-test resourceVersion: "26784" uid: 0d0c0357-abc0-45cc-a4fb-7f69bc343f68 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:40:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-3e303} kind: Pod name: switch-graph-3e303-66c47b978f-nfvht namespace: kserve-ci-e2e-test resourceVersion: "26748" uid: 14cab6c0-a554-416d-ae2e-76b2c089e696 kind: Event lastTimestamp: "2026-05-20T14:40:47Z" message: Stopping container switch-graph-3e303 metadata: creationTimestamp: "2026-05-20T14:40:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:40:47Z" name: switch-graph-3e303-66c47b978f-nfvht.18b14d0d7358e0d3 namespace: kserve-ci-e2e-test resourceVersion: "33646" uid: 17a89602-78b4-4f6c-a39e-1b5dd0842698 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-20T14:40:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-3e303} kind: Pod name: switch-graph-3e303-66c47b978f-nfvht namespace: kserve-ci-e2e-test resourceVersion: "26748" uid: 14cab6c0-a554-416d-ae2e-76b2c089e696 kind: Event lastTimestamp: "2026-05-20T14:41:11Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-20T14:40:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:41:11Z" name: switch-graph-3e303-66c47b978f-nfvht.18b14d0e45d79e6b namespace: kserve-ci-e2e-test resourceVersion: "33890" uid: d4368b9f-f8d3-4bad-8e90-4b55c7a4eadf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:24:28Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-3e303-66c47b978f namespace: kserve-ci-e2e-test resourceVersion: "26743" uid: f91cf58b-cfb0-4a92-bee3-fd4bf33dc3af kind: Event lastTimestamp: "2026-05-20T14:24:28Z" message: 'Created pod: switch-graph-3e303-66c47b978f-nfvht' metadata: creationTimestamp: "2026-05-20T14:24:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:24:28Z" name: switch-graph-3e303-66c47b978f.18b14c297048fec6 namespace: kserve-ci-e2e-test resourceVersion: "26749" uid: d68536dc-0166-43f6-bc37-4805fe17dd7b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:24:28Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-3e303 namespace: kserve-ci-e2e-test resourceVersion: "26741" uid: 95206eb1-88c4-4441-aafb-1e6995d9872e kind: Event lastTimestamp: "2026-05-20T14:24:28Z" message: Scaled up replica set switch-graph-3e303-66c47b978f from 0 to 1 metadata: creationTimestamp: "2026-05-20T14:24:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:24:28Z" name: switch-graph-3e303.18b14c296f1ed1da namespace: kserve-ci-e2e-test resourceVersion: "26744" uid: a9d061c6-cd0e-4ae6-b0c0-5eede2f3437a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:24:28Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-3e303 namespace: kserve-ci-e2e-test resourceVersion: "26760" uid: f1d95446-fd43-4f76-a755-3b61c936a6db kind: Event lastTimestamp: "2026-05-20T14:24:28Z" message: InferenceGraph [switch-graph-3e303] is Ready metadata: creationTimestamp: "2026-05-20T14:24:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:24:28Z" name: switch-graph-3e303.18b14c29762c44bb namespace: kserve-ci-e2e-test resourceVersion: "26762" uid: 77baed18-3f28-4740-be64-a2b14c265c77 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:24:28Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-3e303 namespace: kserve-ci-e2e-test resourceVersion: "26734" uid: f1d95446-fd43-4f76-a755-3b61c936a6db kind: Event lastTimestamp: "2026-05-20T14:24:28Z" message: 'Failed to update status for InferenceGraph "switch-graph-3e303": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-3e303": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T14:24:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:24:28Z" name: switch-graph-3e303.18b14c2979354cb1 namespace: kserve-ci-e2e-test resourceVersion: "26764" uid: 2119b3f2-e6cd-4656-8635-fb6696513f5f reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:24:28Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-3e303 namespace: kserve-ci-e2e-test resourceVersion: "26734" uid: f1d95446-fd43-4f76-a755-3b61c936a6db kind: Event lastTimestamp: "2026-05-20T14:24:28Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-3e303": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T14:24:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:24:28Z" name: switch-graph-3e303.18b14c297935a355 namespace: kserve-ci-e2e-test resourceVersion: "26765" uid: 36da386e-f21e-479e-bd81-cccb6c7bcf85 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T14:24:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-3e303 namespace: kserve-ci-e2e-test resourceVersion: "26742" uid: e64f0b87-a258-48c7-bd61-57374757d21d kind: Event lastTimestamp: "2026-05-20T14:24:58Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:24:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:24:58Z" name: switch-graph-3e303.18b14c2ceda1b273 namespace: kserve-ci-e2e-test resourceVersion: "27046" uid: 1d10a9d6-8efa-4674-bd9d-6cc685259221 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T14:24:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-3e303 namespace: kserve-ci-e2e-test resourceVersion: "26742" uid: e64f0b87-a258-48c7-bd61-57374757d21d kind: Event lastTimestamp: "2026-05-20T14:24:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:24:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:24:58Z" name: switch-graph-3e303.18b14c2cedf28a9c namespace: kserve-ci-e2e-test resourceVersion: "27047" uid: f2dfdf41-66eb-46ef-ae65-1ae173accad3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-05-20T14:03:08.250141Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-69925-7c7b5bdf4d-g6qn6 namespace: kserve-ci-e2e-test resourceVersion: "15472" uid: ccbd9ee3-f4d3-4d14-970a-257df2ffb815 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-69925-7c7b5bdf4d-g6qn6 to ip-10-0-136-75.ec2.internal metadata: creationTimestamp: "2026-05-20T14:03:08Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T14:03:08Z" name: switch-graph-69925-7c7b5bdf4d-g6qn6.18b14aff528d07ef namespace: kserve-ci-e2e-test resourceVersion: "15478" uid: 469a77ef-3177-441c-8ad0-a32732fa3442 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-747dc7bb84-jx2kx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:08Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-69925-7c7b5bdf4d-g6qn6 namespace: kserve-ci-e2e-test resourceVersion: "15476" uid: ccbd9ee3-f4d3-4d14-970a-257df2ffb815 kind: Event lastTimestamp: "2026-05-20T14:03:08Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-69925-serving-cert" not found' metadata: creationTimestamp: "2026-05-20T14:03:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:08Z" name: switch-graph-69925-7c7b5bdf4d-g6qn6.18b14aff5ea41f19 namespace: kserve-ci-e2e-test resourceVersion: "15494" uid: 8e191312-83d4-4df8-879d-62b3433e1a8b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:09Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-69925-7c7b5bdf4d-g6qn6 namespace: kserve-ci-e2e-test resourceVersion: "15482" uid: ccbd9ee3-f4d3-4d14-970a-257df2ffb815 kind: Event lastTimestamp: "2026-05-20T14:03:09Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T14:03:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T14:03:09Z" name: switch-graph-69925-7c7b5bdf4d-g6qn6.18b14aff8ff54360 namespace: kserve-ci-e2e-test resourceVersion: "15507" uid: 027b6893-051b-40a5-ad48-ff95a7b2ad8b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-69925} kind: Pod name: switch-graph-69925-7c7b5bdf4d-g6qn6 namespace: kserve-ci-e2e-test resourceVersion: "15476" uid: ccbd9ee3-f4d3-4d14-970a-257df2ffb815 kind: Event lastTimestamp: "2026-05-20T14:03:09Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:04926a9d7185ff179745ae32ed77ae962904508dbcb7541e9bf600392af6f503" metadata: creationTimestamp: "2026-05-20T14:03:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:09Z" name: switch-graph-69925-7c7b5bdf4d-g6qn6.18b14aff919caef7 namespace: kserve-ci-e2e-test resourceVersion: "15509" uid: 4616a498-1d0c-4fd5-a1e0-02e525672447 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-69925} kind: Pod name: switch-graph-69925-7c7b5bdf4d-g6qn6 namespace: kserve-ci-e2e-test resourceVersion: "15476" uid: ccbd9ee3-f4d3-4d14-970a-257df2ffb815 kind: Event lastTimestamp: "2026-05-20T14:03:11Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:04926a9d7185ff179745ae32ed77ae962904508dbcb7541e9bf600392af6f503" in 2.201s (2.201s including waiting). Image size: 218002168 bytes.' metadata: creationTimestamp: "2026-05-20T14:03:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:11Z" name: switch-graph-69925-7c7b5bdf4d-g6qn6.18b14b0014d44a6b namespace: kserve-ci-e2e-test resourceVersion: "15526" uid: 2cf1e82a-0425-44e9-b42e-1b99cd8c63c5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-69925} kind: Pod name: switch-graph-69925-7c7b5bdf4d-g6qn6 namespace: kserve-ci-e2e-test resourceVersion: "15476" uid: ccbd9ee3-f4d3-4d14-970a-257df2ffb815 kind: Event lastTimestamp: "2026-05-20T14:03:11Z" message: 'Created container: switch-graph-69925' metadata: creationTimestamp: "2026-05-20T14:03:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:11Z" name: switch-graph-69925-7c7b5bdf4d-g6qn6.18b14b001841fa15 namespace: kserve-ci-e2e-test resourceVersion: "15527" uid: db8519a7-ee84-4765-974e-b289b15340c9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-69925} kind: Pod name: switch-graph-69925-7c7b5bdf4d-g6qn6 namespace: kserve-ci-e2e-test resourceVersion: "15476" uid: ccbd9ee3-f4d3-4d14-970a-257df2ffb815 kind: Event lastTimestamp: "2026-05-20T14:03:11Z" message: Started container switch-graph-69925 metadata: creationTimestamp: "2026-05-20T14:03:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:11Z" name: switch-graph-69925-7c7b5bdf4d-g6qn6.18b14b0018b1a322 namespace: kserve-ci-e2e-test resourceVersion: "15528" uid: 4e962480-28c7-4b03-93bb-ec78941d0083 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-69925} kind: Pod name: switch-graph-69925-7c7b5bdf4d-g6qn6 namespace: kserve-ci-e2e-test resourceVersion: "15476" uid: ccbd9ee3-f4d3-4d14-970a-257df2ffb815 kind: Event lastTimestamp: "2026-05-20T14:03:18Z" message: Stopping container switch-graph-69925 metadata: creationTimestamp: "2026-05-20T14:03:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:18Z" name: switch-graph-69925-7c7b5bdf4d-g6qn6.18b14b01aacdbd02 namespace: kserve-ci-e2e-test resourceVersion: "15620" uid: bb324201-d68f-4cbd-92ad-c9df1b031bd1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-20T14:03:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-69925} kind: Pod name: switch-graph-69925-7c7b5bdf4d-g6qn6 namespace: kserve-ci-e2e-test resourceVersion: "15476" uid: ccbd9ee3-f4d3-4d14-970a-257df2ffb815 kind: Event lastTimestamp: "2026-05-20T14:03:47Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-20T14:03:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:47Z" name: switch-graph-69925-7c7b5bdf4d-g6qn6.18b14b02be898f97 namespace: kserve-ci-e2e-test resourceVersion: "16036" uid: 917ededd-cc05-4774-8442-edfbd955425e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:08Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-69925-7c7b5bdf4d namespace: kserve-ci-e2e-test resourceVersion: "15469" uid: f7805a59-f9b9-422d-bd98-95ea1ecaaf39 kind: Event lastTimestamp: "2026-05-20T14:03:08Z" message: 'Created pod: switch-graph-69925-7c7b5bdf4d-g6qn6' metadata: creationTimestamp: "2026-05-20T14:03:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:03:08Z" name: switch-graph-69925-7c7b5bdf4d.18b14aff51f6a5f6 namespace: kserve-ci-e2e-test resourceVersion: "15475" uid: ac88f619-f896-4e48-bdc2-75a83bc88435 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:08Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-69925 namespace: kserve-ci-e2e-test resourceVersion: "15467" uid: 1baed0aa-231f-4fcc-b9d5-b9f2be0267ac kind: Event lastTimestamp: "2026-05-20T14:03:08Z" message: Scaled up replica set switch-graph-69925-7c7b5bdf4d from 0 to 1 metadata: creationTimestamp: "2026-05-20T14:03:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:03:08Z" name: switch-graph-69925.18b14aff5097cabc namespace: kserve-ci-e2e-test resourceVersion: "15471" uid: 6cd0e4bc-8abe-4b19-baa2-ffbb40314517 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:08Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-69925 namespace: kserve-ci-e2e-test resourceVersion: "15486" uid: fd0ae952-2e8e-4c18-b7fb-2bf776937f37 kind: Event lastTimestamp: "2026-05-20T14:03:08Z" message: InferenceGraph [switch-graph-69925] is Ready metadata: creationTimestamp: "2026-05-20T14:03:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:03:08Z" name: switch-graph-69925.18b14aff57ae33fb namespace: kserve-ci-e2e-test resourceVersion: "15488" uid: 9e5aa138-f704-4e67-8eb5-b63a37dc25ce reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T14:04:18.597859Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-83116-6fdf4466fc-m5mtv namespace: kserve-ci-e2e-test resourceVersion: "16494" uid: 096f5b5c-1a00-455e-8523-127f66e27ce5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-83116-6fdf4466fc-m5mtv to ip-10-0-136-75.ec2.internal metadata: creationTimestamp: "2026-05-20T14:04:18Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T14:04:18Z" name: switch-graph-83116-6fdf4466fc-m5mtv.18b14b0fb39a0a40 namespace: kserve-ci-e2e-test resourceVersion: "16502" uid: 99d99b8e-c629-43c7-acc8-1d7f1d9b7406 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-747dc7bb84-jx2kx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:04:18Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-83116-6fdf4466fc-m5mtv namespace: kserve-ci-e2e-test resourceVersion: "16499" uid: 096f5b5c-1a00-455e-8523-127f66e27ce5 kind: Event lastTimestamp: "2026-05-20T14:04:18Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-83116-serving-cert" not found' metadata: creationTimestamp: "2026-05-20T14:04:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:04:18Z" name: switch-graph-83116-6fdf4466fc-m5mtv.18b14b0fbd86dc08 namespace: kserve-ci-e2e-test resourceVersion: "16517" uid: 860ec4eb-98b0-4f68-a01f-d461583bdb13 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:04:19Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-83116-6fdf4466fc-m5mtv namespace: kserve-ci-e2e-test resourceVersion: "16505" uid: 096f5b5c-1a00-455e-8523-127f66e27ce5 kind: Event lastTimestamp: "2026-05-20T14:04:19Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T14:04:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T14:04:19Z" name: switch-graph-83116-6fdf4466fc-m5mtv.18b14b0ff1870591 namespace: kserve-ci-e2e-test resourceVersion: "16528" uid: f0bcae54-432a-4950-a2d5-c226bddb10ae reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:04:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-83116} kind: Pod name: switch-graph-83116-6fdf4466fc-m5mtv namespace: kserve-ci-e2e-test resourceVersion: "16499" uid: 096f5b5c-1a00-455e-8523-127f66e27ce5 kind: Event lastTimestamp: "2026-05-20T14:04:19Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:04926a9d7185ff179745ae32ed77ae962904508dbcb7541e9bf600392af6f503" already present on machine metadata: creationTimestamp: "2026-05-20T14:04:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:04:19Z" name: switch-graph-83116-6fdf4466fc-m5mtv.18b14b0ff2e958b1 namespace: kserve-ci-e2e-test resourceVersion: "16530" uid: 594f9a32-f35a-47a4-8674-0bfe1263b553 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:04:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-83116} kind: Pod name: switch-graph-83116-6fdf4466fc-m5mtv namespace: kserve-ci-e2e-test resourceVersion: "16499" uid: 096f5b5c-1a00-455e-8523-127f66e27ce5 kind: Event lastTimestamp: "2026-05-20T14:04:19Z" message: 'Created container: switch-graph-83116' metadata: creationTimestamp: "2026-05-20T14:04:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:04:19Z" name: switch-graph-83116-6fdf4466fc-m5mtv.18b14b0ff5ee1c0e namespace: kserve-ci-e2e-test resourceVersion: "16532" uid: 9ed5fe57-f83f-4ca7-9999-009b8316f123 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:04:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-83116} kind: Pod name: switch-graph-83116-6fdf4466fc-m5mtv namespace: kserve-ci-e2e-test resourceVersion: "16499" uid: 096f5b5c-1a00-455e-8523-127f66e27ce5 kind: Event lastTimestamp: "2026-05-20T14:04:19Z" message: Started container switch-graph-83116 metadata: creationTimestamp: "2026-05-20T14:04:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:04:19Z" name: switch-graph-83116-6fdf4466fc-m5mtv.18b14b0ff658e8a0 namespace: kserve-ci-e2e-test resourceVersion: "16533" uid: 2df86f65-c087-4c4c-bf52-ed40a4beb73b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:12:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-83116} kind: Pod name: switch-graph-83116-6fdf4466fc-m5mtv namespace: kserve-ci-e2e-test resourceVersion: "16499" uid: 096f5b5c-1a00-455e-8523-127f66e27ce5 kind: Event lastTimestamp: "2026-05-20T14:12:33Z" message: Stopping container switch-graph-83116 metadata: creationTimestamp: "2026-05-20T14:12:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:12:33Z" name: switch-graph-83116-6fdf4466fc-m5mtv.18b14b82e698db65 namespace: kserve-ci-e2e-test resourceVersion: "19992" uid: 7881ba46-ee5e-480a-9960-bb7459c8735c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-20T14:12:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-83116} kind: Pod name: switch-graph-83116-6fdf4466fc-m5mtv namespace: kserve-ci-e2e-test resourceVersion: "16499" uid: 096f5b5c-1a00-455e-8523-127f66e27ce5 kind: Event lastTimestamp: "2026-05-20T14:13:01Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-20T14:12:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:13:01Z" name: switch-graph-83116-6fdf4466fc-m5mtv.18b14b838ea44384 namespace: kserve-ci-e2e-test resourceVersion: "20322" uid: ef61501a-eb98-47a9-ac62-fd02a7813732 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-75.ec2.internal source: component: kubelet host: ip-10-0-136-75.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:04:18Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-83116-6fdf4466fc namespace: kserve-ci-e2e-test resourceVersion: "16488" uid: d6ac2f67-2215-447e-b5e6-bb1a7459177c kind: Event lastTimestamp: "2026-05-20T14:04:18Z" message: 'Created pod: switch-graph-83116-6fdf4466fc-m5mtv' metadata: creationTimestamp: "2026-05-20T14:04:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:04:18Z" name: switch-graph-83116-6fdf4466fc.18b14b0fb2fa8427 namespace: kserve-ci-e2e-test resourceVersion: "16497" uid: f19d3259-d09b-4c85-93d3-2806eb1ff43f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:04:18Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-83116 namespace: kserve-ci-e2e-test resourceVersion: "16486" uid: 9369def5-a6cb-4cf1-a973-f0e9f05ec091 kind: Event lastTimestamp: "2026-05-20T14:04:18Z" message: Scaled up replica set switch-graph-83116-6fdf4466fc from 0 to 1 metadata: creationTimestamp: "2026-05-20T14:04:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:04:18Z" name: switch-graph-83116.18b14b0fb1e59342 namespace: kserve-ci-e2e-test resourceVersion: "16491" uid: fc2503dc-f65b-4ab4-ac23-a2e3aca93c02 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:04:18Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-83116 namespace: kserve-ci-e2e-test resourceVersion: "16507" uid: 670d3d7e-f3f2-4a8c-9ace-3ada81edc4ce kind: Event lastTimestamp: "2026-05-20T14:04:18Z" message: InferenceGraph [switch-graph-83116] is Ready metadata: creationTimestamp: "2026-05-20T14:04:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:04:18Z" name: switch-graph-83116.18b14b0fb5fe0dc7 namespace: kserve-ci-e2e-test resourceVersion: "16510" uid: 31c765d2-a33a-4aef-9112-669023127e78 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T14:04:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-83116 namespace: kserve-ci-e2e-test resourceVersion: "16487" uid: ca137532-341c-4f4b-88a8-d0bf1a892b14 kind: Event lastTimestamp: "2026-05-20T14:04:48Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:04:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:04:48Z" name: switch-graph-83116.18b14b133161ee5e namespace: kserve-ci-e2e-test resourceVersion: "16790" uid: 0a88d250-daa7-4ad4-aad3-c35218ffd17a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T14:04:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-83116 namespace: kserve-ci-e2e-test resourceVersion: "16487" uid: ca137532-341c-4f4b-88a8-d0bf1a892b14 kind: Event lastTimestamp: "2026-05-20T14:04:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:04:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:04:48Z" name: switch-graph-83116.18b14b1331eb7d4c namespace: kserve-ci-e2e-test resourceVersion: "16791" uid: c77b3022-f700-4a00-9ddf-4ad52fb7f10c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning kind: EventList metadata: resourceVersion: "33903"