--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:27:35.780590Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-ccdbb-59bfb89b8-8swsg namespace: kserve-ci-e2e-test resourceVersion: "20810" uid: c1406e5f-cf63-4a83-b38c-0513f18af20a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-ccdbb-59bfb89b8-8swsg to ip-10-0-128-59.ec2.internal metadata: creationTimestamp: "2026-04-16T18:27:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:27:35Z" name: ensemble-graph-ccdbb-59bfb89b8-8swsg.18a6e9b27893221d namespace: kserve-ci-e2e-test resourceVersion: "20815" uid: d0aca0fb-bdc0-40cc-a20b-39d96fafa588 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4b7bbd69-nffhf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:36Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-ccdbb-59bfb89b8-8swsg namespace: kserve-ci-e2e-test resourceVersion: "20811" uid: c1406e5f-cf63-4a83-b38c-0513f18af20a kind: Event lastTimestamp: "2026-04-16T18:27:36Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-ccdbb-serving-cert" not found' metadata: creationTimestamp: "2026-04-16T18:27:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:27:36Z" name: ensemble-graph-ccdbb-59bfb89b8-8swsg.18a6e9b287a47e69 namespace: kserve-ci-e2e-test resourceVersion: "20827" uid: 23a7bbc1-b7c2-4482-9a64-6fc5cdc491d4 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:36Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-ccdbb-59bfb89b8-8swsg namespace: kserve-ci-e2e-test resourceVersion: "20819" uid: c1406e5f-cf63-4a83-b38c-0513f18af20a kind: Event lastTimestamp: "2026-04-16T18:27:36Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:27:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:27:36Z" name: ensemble-graph-ccdbb-59bfb89b8-8swsg.18a6e9b2b55a7f32 namespace: kserve-ci-e2e-test resourceVersion: "20835" uid: 2f3bac6d-556f-4d43-8e11-2754cd7f36dd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-ccdbb} kind: Pod name: ensemble-graph-ccdbb-59bfb89b8-8swsg namespace: kserve-ci-e2e-test resourceVersion: "20811" uid: c1406e5f-cf63-4a83-b38c-0513f18af20a kind: Event lastTimestamp: "2026-04-16T18:27:36Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:6060da2148ba78c06d6de96222351f1e5f7d3ada4027ac02142aa0e1928e4f29" already present on machine metadata: creationTimestamp: "2026-04-16T18:27:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:27:36Z" name: ensemble-graph-ccdbb-59bfb89b8-8swsg.18a6e9b2b67fd78c namespace: kserve-ci-e2e-test resourceVersion: "20837" uid: 758f8b13-134d-4aa7-9ec0-b9801a512348 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-ccdbb} kind: Pod name: ensemble-graph-ccdbb-59bfb89b8-8swsg namespace: kserve-ci-e2e-test resourceVersion: "20811" uid: c1406e5f-cf63-4a83-b38c-0513f18af20a kind: Event lastTimestamp: "2026-04-16T18:27:36Z" message: 'Created container: ensemble-graph-ccdbb' metadata: creationTimestamp: "2026-04-16T18:27:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:27:36Z" name: ensemble-graph-ccdbb-59bfb89b8-8swsg.18a6e9b2ba1ce090 namespace: kserve-ci-e2e-test resourceVersion: "20839" uid: d208c12a-6671-4b59-bfa6-69602c6c9927 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-ccdbb} kind: Pod name: ensemble-graph-ccdbb-59bfb89b8-8swsg namespace: kserve-ci-e2e-test resourceVersion: "20811" uid: c1406e5f-cf63-4a83-b38c-0513f18af20a kind: Event lastTimestamp: "2026-04-16T18:27:36Z" message: Started container ensemble-graph-ccdbb metadata: creationTimestamp: "2026-04-16T18:27:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:27:36Z" name: ensemble-graph-ccdbb-59bfb89b8-8swsg.18a6e9b2ba8c78c4 namespace: kserve-ci-e2e-test resourceVersion: "20840" uid: bebf307b-bf8a-4bbd-b515-fe4d58fa8d4e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-ccdbb} kind: Pod name: ensemble-graph-ccdbb-59bfb89b8-8swsg namespace: kserve-ci-e2e-test resourceVersion: "20811" uid: c1406e5f-cf63-4a83-b38c-0513f18af20a kind: Event lastTimestamp: "2026-04-16T18:27:45Z" message: Stopping container ensemble-graph-ccdbb metadata: creationTimestamp: "2026-04-16T18:27:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:27:45Z" name: ensemble-graph-ccdbb-59bfb89b8-8swsg.18a6e9b4d068313d namespace: kserve-ci-e2e-test resourceVersion: "20938" uid: 8d54ab73-ef9b-4a52-9786-3d52b0d910e8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:27:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-ccdbb} kind: Pod name: ensemble-graph-ccdbb-59bfb89b8-8swsg namespace: kserve-ci-e2e-test resourceVersion: "20811" uid: c1406e5f-cf63-4a83-b38c-0513f18af20a kind: Event lastTimestamp: "2026-04-16T18:28:12Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T18:27:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:28:12Z" name: ensemble-graph-ccdbb-59bfb89b8-8swsg.18a6e9b54f65ad11 namespace: kserve-ci-e2e-test resourceVersion: "21319" uid: 5b13d083-b924-4034-b331-8c0ddcc8c4cd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-ccdbb-59bfb89b8 namespace: kserve-ci-e2e-test resourceVersion: "20807" uid: c8486b1d-194c-4798-9431-4171b164406a kind: Event lastTimestamp: "2026-04-16T18:27:35Z" message: 'Created pod: ensemble-graph-ccdbb-59bfb89b8-8swsg' metadata: creationTimestamp: "2026-04-16T18:27:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:27:35Z" name: ensemble-graph-ccdbb-59bfb89b8.18a6e9b278314e8b namespace: kserve-ci-e2e-test resourceVersion: "20812" uid: fd841e42-5c2b-47eb-b51a-9f5c42dce780 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-ccdbb namespace: kserve-ci-e2e-test resourceVersion: "20805" uid: fe75856e-8f23-431e-b922-7897d45df6c8 kind: Event lastTimestamp: "2026-04-16T18:27:35Z" message: Scaled up replica set ensemble-graph-ccdbb-59bfb89b8 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:27:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:27:35Z" name: ensemble-graph-ccdbb.18a6e9b277731bc1 namespace: kserve-ci-e2e-test resourceVersion: "20808" uid: 30b60dd4-2316-4ceb-a46b-c9e35f4cf09a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:35Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-ccdbb namespace: kserve-ci-e2e-test resourceVersion: "20821" uid: b3bf9b25-12e8-4b14-9f3e-65de583e342d kind: Event lastTimestamp: "2026-04-16T18:27:35Z" message: InferenceGraph [ensemble-graph-ccdbb] is Ready metadata: creationTimestamp: "2026-04-16T18:27:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:27:35Z" name: ensemble-graph-ccdbb.18a6e9b27b15fcfc namespace: kserve-ci-e2e-test resourceVersion: "20823" uid: c6252c3e-ec44-4a1e-bdf1-b4e422217c96 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:35Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-ccdbb namespace: kserve-ci-e2e-test resourceVersion: "20797" uid: b3bf9b25-12e8-4b14-9f3e-65de583e342d kind: Event lastTimestamp: "2026-04-16T18:27:35Z" message: 'Failed to update status for InferenceGraph "ensemble-graph-ccdbb": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "ensemble-graph-ccdbb": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:27:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:27:35Z" name: ensemble-graph-ccdbb.18a6e9b27dd5eda1 namespace: kserve-ci-e2e-test resourceVersion: "20824" uid: f50f4651-32e5-412f-9106-136c70a9b3c2 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:35Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-ccdbb namespace: kserve-ci-e2e-test resourceVersion: "20797" uid: b3bf9b25-12e8-4b14-9f3e-65de583e342d kind: Event lastTimestamp: "2026-04-16T18:27:35Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "ensemble-graph-ccdbb": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:27:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:27:35Z" name: ensemble-graph-ccdbb.18a6e9b27dd6363e namespace: kserve-ci-e2e-test resourceVersion: "20825" uid: 8fc8ba0f-c438-452b-a9a2-62741256d8f8 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:28:56.106480Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-df772-7fb98745c-jlqxb namespace: kserve-ci-e2e-test resourceVersion: "21826" uid: c062514c-adc1-4f8f-ad51-7a656f69c3eb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-df772-7fb98745c-jlqxb to ip-10-0-128-59.ec2.internal metadata: creationTimestamp: "2026-04-16T18:28:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:28:56Z" name: ensemble-graph-df772-7fb98745c-jlqxb.18a6e9c52c5eef73 namespace: kserve-ci-e2e-test resourceVersion: "21831" uid: 6be5a7db-b27a-4576-8e3b-aeb58c387d33 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4b7bbd69-nffhf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:56Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-df772-7fb98745c-jlqxb namespace: kserve-ci-e2e-test resourceVersion: "21835" uid: c062514c-adc1-4f8f-ad51-7a656f69c3eb kind: Event lastTimestamp: "2026-04-16T18:28:56Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:28:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:28:56Z" name: ensemble-graph-df772-7fb98745c-jlqxb.18a6e9c5456c0ae6 namespace: kserve-ci-e2e-test resourceVersion: "21846" uid: ef3ae4b2-1b4b-46e5-819f-13356b6f309b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-df772} kind: Pod name: ensemble-graph-df772-7fb98745c-jlqxb namespace: kserve-ci-e2e-test resourceVersion: "21828" uid: c062514c-adc1-4f8f-ad51-7a656f69c3eb kind: Event lastTimestamp: "2026-04-16T18:28:56Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:6060da2148ba78c06d6de96222351f1e5f7d3ada4027ac02142aa0e1928e4f29" already present on machine metadata: creationTimestamp: "2026-04-16T18:28:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:28:56Z" name: ensemble-graph-df772-7fb98745c-jlqxb.18a6e9c54698b06f namespace: kserve-ci-e2e-test resourceVersion: "21849" uid: 7c2402cf-cd5d-4177-94a0-059d7eaf124f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-df772} kind: Pod name: ensemble-graph-df772-7fb98745c-jlqxb namespace: kserve-ci-e2e-test resourceVersion: "21828" uid: c062514c-adc1-4f8f-ad51-7a656f69c3eb kind: Event lastTimestamp: "2026-04-16T18:28:56Z" message: 'Created container: ensemble-graph-df772' metadata: creationTimestamp: "2026-04-16T18:28:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:28:56Z" name: ensemble-graph-df772-7fb98745c-jlqxb.18a6e9c549c2ce0c namespace: kserve-ci-e2e-test resourceVersion: "21850" uid: 362a959f-381f-4d68-b7b0-d366a1a6a393 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-df772} kind: Pod name: ensemble-graph-df772-7fb98745c-jlqxb namespace: kserve-ci-e2e-test resourceVersion: "21828" uid: c062514c-adc1-4f8f-ad51-7a656f69c3eb kind: Event lastTimestamp: "2026-04-16T18:28:56Z" message: Started container ensemble-graph-df772 metadata: creationTimestamp: "2026-04-16T18:28:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:28:56Z" name: ensemble-graph-df772-7fb98745c-jlqxb.18a6e9c54a318bbd namespace: kserve-ci-e2e-test resourceVersion: "21851" uid: ea6ebb58-a1c7-4d36-b671-981a94f1aa32 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-df772} kind: Pod name: ensemble-graph-df772-7fb98745c-jlqxb namespace: kserve-ci-e2e-test resourceVersion: "21828" uid: c062514c-adc1-4f8f-ad51-7a656f69c3eb kind: Event lastTimestamp: "2026-04-16T18:37:10Z" message: Stopping container ensemble-graph-df772 metadata: creationTimestamp: "2026-04-16T18:37:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:37:10Z" name: ensemble-graph-df772-7fb98745c-jlqxb.18a6ea38588404aa namespace: kserve-ci-e2e-test resourceVersion: "25613" uid: 1f8bdb95-6ac6-4e28-b5a7-2754fac928c7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:37:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-df772} kind: Pod name: ensemble-graph-df772-7fb98745c-jlqxb namespace: kserve-ci-e2e-test resourceVersion: "21828" uid: c062514c-adc1-4f8f-ad51-7a656f69c3eb kind: Event lastTimestamp: "2026-04-16T18:37:38Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T18:37:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:37:38Z" name: ensemble-graph-df772-7fb98745c-jlqxb.18a6ea38eaf8767c namespace: kserve-ci-e2e-test resourceVersion: "25928" uid: 7985751d-f3d3-4e84-a29a-22e2e55d83e9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-df772-7fb98745c namespace: kserve-ci-e2e-test resourceVersion: "21823" uid: 9ade7493-3ddf-4ef7-a2c2-4c544545284f kind: Event lastTimestamp: "2026-04-16T18:28:56Z" message: 'Created pod: ensemble-graph-df772-7fb98745c-jlqxb' metadata: creationTimestamp: "2026-04-16T18:28:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:28:56Z" name: ensemble-graph-df772-7fb98745c.18a6e9c52bf14a94 namespace: kserve-ci-e2e-test resourceVersion: "21827" uid: 8e0c8055-e50a-4904-8da5-8ac9dd20088d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-df772 namespace: kserve-ci-e2e-test resourceVersion: "21821" uid: 6cdd30f3-96d1-489d-9747-879d5135ae09 kind: Event lastTimestamp: "2026-04-16T18:28:56Z" message: Scaled up replica set ensemble-graph-df772-7fb98745c from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:28:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:28:56Z" name: ensemble-graph-df772.18a6e9c52b156d00 namespace: kserve-ci-e2e-test resourceVersion: "21824" uid: 9be108db-d692-41f2-8e36-5f5710abe04c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:56Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-df772 namespace: kserve-ci-e2e-test resourceVersion: "21838" uid: 5aac54db-2072-4846-90dd-a0192ea4750c kind: Event lastTimestamp: "2026-04-16T18:28:56Z" message: InferenceGraph [ensemble-graph-df772] is Ready metadata: creationTimestamp: "2026-04-16T18:28:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:28:56Z" name: ensemble-graph-df772.18a6e9c52f704f24 namespace: kserve-ci-e2e-test resourceVersion: "21840" uid: d8cd0ab8-e258-4ae7-8675-6b55404a911a reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:29:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-df772 namespace: kserve-ci-e2e-test resourceVersion: "21822" uid: 69910f62-18e3-4957-bf6c-84e8cd13a7e8 kind: Event lastTimestamp: "2026-04-16T18:29:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:29:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:29:11Z" name: ensemble-graph-df772.18a6e9c8aa3e80bd namespace: kserve-ci-e2e-test resourceVersion: "21985" uid: 7c63dd3d-7f18-4fe7-b324-593f3f95cf6c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:29:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-df772 namespace: kserve-ci-e2e-test resourceVersion: "21822" uid: 69910f62-18e3-4957-bf6c-84e8cd13a7e8 kind: Event lastTimestamp: "2026-04-16T18:29:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:29:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:29:11Z" name: ensemble-graph-df772.18a6e9c8aaa74be9 namespace: kserve-ci-e2e-test resourceVersion: "21987" uid: c4eebfc5-8cf4-4a9e-86fa-fa98491d6e22 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:29:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-df772 namespace: kserve-ci-e2e-test resourceVersion: "21986" uid: 69910f62-18e3-4957-bf6c-84e8cd13a7e8 kind: Event lastTimestamp: "2026-04-16T18:29:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:29:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:29:26Z" name: ensemble-graph-df772.18a6e9cc295ca07b namespace: kserve-ci-e2e-test resourceVersion: "22123" uid: ac8cbd3d-880b-40eb-96a1-ac773c681b80 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:29:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-df772 namespace: kserve-ci-e2e-test resourceVersion: "21986" uid: 69910f62-18e3-4957-bf6c-84e8cd13a7e8 kind: Event lastTimestamp: "2026-04-16T18:29:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:29:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:29:26Z" name: ensemble-graph-df772.18a6e9cc29c56ec8 namespace: kserve-ci-e2e-test resourceVersion: "22125" uid: 085c74ac-8825-4e04-b68e-20817438fa02 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:17:30.700537Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-0403e-predictor-65bc898d66-f8rjw namespace: kserve-ci-e2e-test resourceVersion: "16082" uid: d1b84db2-4bf3-418a-8219-eeca35b69e0a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-0403e-predictor-65bc898d66-f8rjw to ip-10-0-134-133.ec2.internal metadata: creationTimestamp: "2026-04-16T18:17:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:17:30Z" name: error-404-isvc-0403e-predictor-65bc898d66-f8rjw.18a6e92596fe46e6 namespace: kserve-ci-e2e-test resourceVersion: "16089" uid: 7583e3dc-ffe4-4810-aced-25bd78dfb06e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4b7bbd69-nffhf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:17:30Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-0403e-predictor-65bc898d66-f8rjw namespace: kserve-ci-e2e-test resourceVersion: "16097" uid: d1b84db2-4bf3-418a-8219-eeca35b69e0a kind: Event lastTimestamp: "2026-04-16T18:17:30Z" message: Add eth0 [10.132.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:17:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:17:30Z" name: error-404-isvc-0403e-predictor-65bc898d66-f8rjw.18a6e925a06568df namespace: kserve-ci-e2e-test resourceVersion: "16124" uid: 4adf65b2-567d-4b89-a998-efc01b1bdef0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:17:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0403e-predictor-65bc898d66-f8rjw namespace: kserve-ci-e2e-test resourceVersion: "16087" uid: d1b84db2-4bf3-418a-8219-eeca35b69e0a kind: Event lastTimestamp: "2026-04-16T18:17:30Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1412" already present on machine metadata: creationTimestamp: "2026-04-16T18:17:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:17:30Z" name: error-404-isvc-0403e-predictor-65bc898d66-f8rjw.18a6e925a29bead1 namespace: kserve-ci-e2e-test resourceVersion: "16126" uid: b5467076-9f7d-47d1-9c6f-940a9d82c9f0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:17:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0403e-predictor-65bc898d66-f8rjw namespace: kserve-ci-e2e-test resourceVersion: "16087" uid: d1b84db2-4bf3-418a-8219-eeca35b69e0a kind: Event lastTimestamp: "2026-04-16T18:17:31Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:17:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:17:31Z" name: error-404-isvc-0403e-predictor-65bc898d66-f8rjw.18a6e925aabb9875 namespace: kserve-ci-e2e-test resourceVersion: "16129" uid: a5a24574-3503-43cd-b7bb-1d2e45130bb9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:17:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0403e-predictor-65bc898d66-f8rjw namespace: kserve-ci-e2e-test resourceVersion: "16087" uid: d1b84db2-4bf3-418a-8219-eeca35b69e0a kind: Event lastTimestamp: "2026-04-16T18:17:31Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:17:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:17:31Z" name: error-404-isvc-0403e-predictor-65bc898d66-f8rjw.18a6e925ab36e794 namespace: kserve-ci-e2e-test resourceVersion: "16130" uid: d9cb3894-2951-44b0-b70a-246c6f2d2f78 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:17:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0403e-predictor-65bc898d66-f8rjw namespace: kserve-ci-e2e-test resourceVersion: "16087" uid: d1b84db2-4bf3-418a-8219-eeca35b69e0a kind: Event lastTimestamp: "2026-04-16T18:18:12Z" message: 'Readiness probe failed: dial tcp 10.132.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:17:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:18:12Z" name: error-404-isvc-0403e-predictor-65bc898d66-f8rjw.18a6e925d0bbeaf0 namespace: kserve-ci-e2e-test resourceVersion: "16533" uid: a61bdc16-10b3-4b2e-b17b-51b7cd978934 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0403e-predictor-65bc898d66-f8rjw namespace: kserve-ci-e2e-test resourceVersion: "16087" uid: d1b84db2-4bf3-418a-8219-eeca35b69e0a kind: Event lastTimestamp: "2026-04-16T18:26:55Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:26:55Z" name: error-404-isvc-0403e-predictor-65bc898d66-f8rjw.18a6e9a91f7b9871 namespace: kserve-ci-e2e-test resourceVersion: "20409" uid: 68d46f40-7f60-4bd4-9f48-ca8debebff61 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:17:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-0403e-predictor-65bc898d66 namespace: kserve-ci-e2e-test resourceVersion: "16066" uid: afd856b2-4a1a-4cd4-9d9e-7f38388384ce kind: Event lastTimestamp: "2026-04-16T18:17:30Z" message: 'Created pod: error-404-isvc-0403e-predictor-65bc898d66-f8rjw' metadata: creationTimestamp: "2026-04-16T18:17:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:17:30Z" name: error-404-isvc-0403e-predictor-65bc898d66.18a6e925966f84fc namespace: kserve-ci-e2e-test resourceVersion: "16084" uid: c6be3c56-e3c5-413d-a203-2f31acc6825b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:17:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-0403e-predictor namespace: kserve-ci-e2e-test resourceVersion: "16062" uid: 7bb4ffa9-3e91-4194-867a-5a2073b9b365 kind: Event lastTimestamp: "2026-04-16T18:17:30Z" message: Scaled up replica set error-404-isvc-0403e-predictor-65bc898d66 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:17:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:17:30Z" name: error-404-isvc-0403e-predictor.18a6e925943bab14 namespace: kserve-ci-e2e-test resourceVersion: "16068" uid: 62472c66-4716-4993-98cc-34ecd564d62b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T18:17:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0403e-predictor namespace: kserve-ci-e2e-test resourceVersion: "16065" uid: 2d8bcdb6-26d6-4c22-8daf-2d9b71ace0c8 kind: Event lastTimestamp: "2026-04-16T18:18:30Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:17:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:18:30Z" name: error-404-isvc-0403e-predictor.18a6e92912d3f6b9 namespace: kserve-ci-e2e-test resourceVersion: "16696" uid: cefcbacd-7ba8-4faa-8134-5c102fbe2871 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T18:17:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0403e-predictor namespace: kserve-ci-e2e-test resourceVersion: "16065" uid: 2d8bcdb6-26d6-4c22-8daf-2d9b71ace0c8 kind: Event lastTimestamp: "2026-04-16T18:18:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:17:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:18:30Z" name: error-404-isvc-0403e-predictor.18a6e9291318f0c1 namespace: kserve-ci-e2e-test resourceVersion: "16698" uid: aca6ac5d-8595-4fbd-9cba-15fdde937bc1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:17:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0403e namespace: kserve-ci-e2e-test resourceVersion: "16052" uid: be10d712-4520-4b77-a086-e413c71644b0 kind: Event lastTimestamp: "2026-04-16T18:18:22Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:17:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:18:22Z" name: error-404-isvc-0403e.18a6e925943d2eff namespace: kserve-ci-e2e-test resourceVersion: "16639" uid: 4c349245-ed3f-4fef-8064-19ee81db4f3d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0403e namespace: kserve-ci-e2e-test resourceVersion: "16637" uid: be10d712-4520-4b77-a086-e413c71644b0 kind: Event lastTimestamp: "2026-04-16T18:18:22Z" message: InferenceService [error-404-isvc-0403e] is Ready metadata: creationTimestamp: "2026-04-16T18:18:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:18:22Z" name: error-404-isvc-0403e.18a6e931b512cbd5 namespace: kserve-ci-e2e-test resourceVersion: "16638" uid: b40903cc-c38c-46e2-adcf-4e51b75c6a2c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:17:00.753828Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-04c3d-predictor-7c964cdb94-hzrhl namespace: kserve-ci-e2e-test resourceVersion: "15647" uid: 7fab5246-d585-4dd5-90e6-2902819b30a3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-04c3d-predictor-7c964cdb94-hzrhl to ip-10-0-134-133.ec2.internal metadata: creationTimestamp: "2026-04-16T18:17:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:17:00Z" name: error-404-isvc-04c3d-predictor-7c964cdb94-hzrhl.18a6e91e9e07c251 namespace: kserve-ci-e2e-test resourceVersion: "15652" uid: 6c7476ee-543c-43aa-88fa-1101c7d25f4a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4b7bbd69-nffhf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:17:01Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-04c3d-predictor-7c964cdb94-hzrhl namespace: kserve-ci-e2e-test resourceVersion: "15656" uid: 7fab5246-d585-4dd5-90e6-2902819b30a3 kind: Event lastTimestamp: "2026-04-16T18:17:01Z" message: Add eth0 [10.132.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:17:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:17:01Z" name: error-404-isvc-04c3d-predictor-7c964cdb94-hzrhl.18a6e91eb219d577 namespace: kserve-ci-e2e-test resourceVersion: "15676" uid: 5f14b26f-0d3d-4e4d-a1a1-276966636686 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:17:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-04c3d-predictor-7c964cdb94-hzrhl namespace: kserve-ci-e2e-test resourceVersion: "15650" uid: 7fab5246-d585-4dd5-90e6-2902819b30a3 kind: Event lastTimestamp: "2026-04-16T18:17:01Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1412" already present on machine metadata: creationTimestamp: "2026-04-16T18:17:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:17:01Z" name: error-404-isvc-04c3d-predictor-7c964cdb94-hzrhl.18a6e91eb357d733 namespace: kserve-ci-e2e-test resourceVersion: "15678" uid: 66843826-74d3-4cb9-b2a9-866b7238fe2b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:17:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-04c3d-predictor-7c964cdb94-hzrhl namespace: kserve-ci-e2e-test resourceVersion: "15650" uid: 7fab5246-d585-4dd5-90e6-2902819b30a3 kind: Event lastTimestamp: "2026-04-16T18:17:01Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:17:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:17:01Z" name: error-404-isvc-04c3d-predictor-7c964cdb94-hzrhl.18a6e91eba480372 namespace: kserve-ci-e2e-test resourceVersion: "15684" uid: 294ed8b2-148e-4dae-89a2-9700be5c113c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:17:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-04c3d-predictor-7c964cdb94-hzrhl namespace: kserve-ci-e2e-test resourceVersion: "15650" uid: 7fab5246-d585-4dd5-90e6-2902819b30a3 kind: Event lastTimestamp: "2026-04-16T18:17:01Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:17:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:17:01Z" name: error-404-isvc-04c3d-predictor-7c964cdb94-hzrhl.18a6e91ebabce4bd namespace: kserve-ci-e2e-test resourceVersion: "15686" uid: 2dd009df-154c-43ca-9461-130f73be4e8e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:17:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-04c3d-predictor-7c964cdb94-hzrhl namespace: kserve-ci-e2e-test resourceVersion: "15650" uid: 7fab5246-d585-4dd5-90e6-2902819b30a3 kind: Event lastTimestamp: "2026-04-16T18:17:42Z" message: 'Readiness probe failed: dial tcp 10.132.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:17:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:17:42Z" name: error-404-isvc-04c3d-predictor-7c964cdb94-hzrhl.18a6e91ece1f7e99 namespace: kserve-ci-e2e-test resourceVersion: "16243" uid: ca8dc531-0e74-4305-8162-aa149125eb66 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-04c3d-predictor-7c964cdb94-hzrhl namespace: kserve-ci-e2e-test resourceVersion: "15650" uid: 7fab5246-d585-4dd5-90e6-2902819b30a3 kind: Event lastTimestamp: "2026-04-16T18:26:25Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:26:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:26:25Z" name: error-404-isvc-04c3d-predictor-7c964cdb94-hzrhl.18a6e9a22c5c3493 namespace: kserve-ci-e2e-test resourceVersion: "20060" uid: de259177-a494-4c65-bce0-a363cb2a8167 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:17:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-04c3d-predictor-7c964cdb94 namespace: kserve-ci-e2e-test resourceVersion: "15640" uid: fcaffd1f-3377-4dec-9d9e-6e8e4a2fd199 kind: Event lastTimestamp: "2026-04-16T18:17:00Z" message: 'Created pod: error-404-isvc-04c3d-predictor-7c964cdb94-hzrhl' metadata: creationTimestamp: "2026-04-16T18:17:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:17:00Z" name: error-404-isvc-04c3d-predictor-7c964cdb94.18a6e91e9d924338 namespace: kserve-ci-e2e-test resourceVersion: "15649" uid: fa7fd044-fefb-4158-b016-bcd2f45cf23e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:17:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-04c3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15639" uid: 982cb212-a416-454f-98d8-85f8247138b2 kind: Event lastTimestamp: "2026-04-16T18:17:00Z" message: Scaled up replica set error-404-isvc-04c3d-predictor-7c964cdb94 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:17:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:17:00Z" name: error-404-isvc-04c3d-predictor.18a6e91e9c2d3150 namespace: kserve-ci-e2e-test resourceVersion: "15642" uid: 5c89e49e-5b31-4492-84e6-08b2e89e6b0e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:17:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-04c3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15641" uid: 8c9c9802-6408-408f-acdc-a2b14a84032d kind: Event lastTimestamp: "2026-04-16T18:17:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:17:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:17:30Z" name: error-404-isvc-04c3d-predictor.18a6e9221b264ea0 namespace: kserve-ci-e2e-test resourceVersion: "16103" uid: 8f139c3a-5f2d-4f43-a715-14915011e5db reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:17:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-04c3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15641" uid: 8c9c9802-6408-408f-acdc-a2b14a84032d kind: Event lastTimestamp: "2026-04-16T18:17:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:17:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:17:30Z" name: error-404-isvc-04c3d-predictor.18a6e9221b9606cb namespace: kserve-ci-e2e-test resourceVersion: "16104" uid: 22932171-537e-409e-86c9-b048130609a7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:17:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-04c3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15832" uid: 8c9c9802-6408-408f-acdc-a2b14a84032d kind: Event lastTimestamp: "2026-04-16T18:18:15Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:17:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:18:15Z" name: error-404-isvc-04c3d-predictor.18a6e9291a17ec44 namespace: kserve-ci-e2e-test resourceVersion: "16568" uid: ac484e22-e658-4a0f-a5be-80067c660d49 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:17:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-04c3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15832" uid: 8c9c9802-6408-408f-acdc-a2b14a84032d kind: Event lastTimestamp: "2026-04-16T18:18:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:17:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:18:15Z" name: error-404-isvc-04c3d-predictor.18a6e9291a8c662a namespace: kserve-ci-e2e-test resourceVersion: "16569" uid: 0aa1ac9e-cd17-4e23-bd48-e1e4e5f39a8b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T18:17:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-04c3d namespace: kserve-ci-e2e-test resourceVersion: "15626" uid: 9619e0ca-ed70-4232-9f7b-588cc8b09ad2 kind: Event lastTimestamp: "2026-04-16T18:17:52Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:17:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:17:52Z" name: error-404-isvc-04c3d.18a6e91e9c87f5b4 namespace: kserve-ci-e2e-test resourceVersion: "16346" uid: 311bd97b-1af2-4436-87fc-d7acc07897c2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:17:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-04c3d namespace: kserve-ci-e2e-test resourceVersion: "15626" uid: 9619e0ca-ed70-4232-9f7b-588cc8b09ad2 kind: Event lastTimestamp: "2026-04-16T18:17:00Z" message: 'Failed to update status for InferenceService "error-404-isvc-04c3d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-04c3d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:17:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:17:00Z" name: error-404-isvc-04c3d.18a6e91e9ce65e01 namespace: kserve-ci-e2e-test resourceVersion: "15645" uid: f92a6101-9cc6-4360-bdaf-a3ae7390c123 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:17:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-04c3d namespace: kserve-ci-e2e-test resourceVersion: "15626" uid: 9619e0ca-ed70-4232-9f7b-588cc8b09ad2 kind: Event lastTimestamp: "2026-04-16T18:17:00Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-04c3d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:17:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:17:00Z" name: error-404-isvc-04c3d.18a6e91e9ce6cf0b namespace: kserve-ci-e2e-test resourceVersion: "15648" uid: 9f726e93-305d-4689-9cbe-f09aa23a1b3c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:17:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-04c3d namespace: kserve-ci-e2e-test resourceVersion: "16344" uid: 9619e0ca-ed70-4232-9f7b-588cc8b09ad2 kind: Event lastTimestamp: "2026-04-16T18:17:52Z" message: InferenceService [error-404-isvc-04c3d] is Ready metadata: creationTimestamp: "2026-04-16T18:17:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:17:52Z" name: error-404-isvc-04c3d.18a6e92ab2dafcec namespace: kserve-ci-e2e-test resourceVersion: "16345" uid: 5a1bc20a-c605-4119-9a2e-2ae7fa5374c5 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:15:30.777727Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-390f9-predictor-666f56949b-vxbx4 namespace: kserve-ci-e2e-test resourceVersion: "14624" uid: 3db464a8-101f-4abb-84dc-c2dd96792242 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-390f9-predictor-666f56949b-vxbx4 to ip-10-0-134-133.ec2.internal metadata: creationTimestamp: "2026-04-16T18:15:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:15:30Z" name: error-404-isvc-390f9-predictor-666f56949b-vxbx4.18a6e909ab096600 namespace: kserve-ci-e2e-test resourceVersion: "14633" uid: 2c4f5240-4125-4cdb-b515-21699a175431 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4b7bbd69-nffhf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:15:31Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-390f9-predictor-666f56949b-vxbx4 namespace: kserve-ci-e2e-test resourceVersion: "14627" uid: 3db464a8-101f-4abb-84dc-c2dd96792242 kind: Event lastTimestamp: "2026-04-16T18:15:31Z" message: Add eth0 [10.132.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:15:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:15:31Z" name: error-404-isvc-390f9-predictor-666f56949b-vxbx4.18a6e909bde7c489 namespace: kserve-ci-e2e-test resourceVersion: "14660" uid: 25a04c24-60e8-4280-898c-b3f64dc94edb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:15:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-390f9-predictor-666f56949b-vxbx4 namespace: kserve-ci-e2e-test resourceVersion: "14627" uid: 3db464a8-101f-4abb-84dc-c2dd96792242 kind: Event lastTimestamp: "2026-04-16T18:15:31Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1412" metadata: creationTimestamp: "2026-04-16T18:15:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:15:31Z" name: error-404-isvc-390f9-predictor-666f56949b-vxbx4.18a6e909c12b62ba namespace: kserve-ci-e2e-test resourceVersion: "14675" uid: 0f9719a2-2b32-4c32-a63d-b48b5caea641 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:15:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-390f9-predictor-666f56949b-vxbx4 namespace: kserve-ci-e2e-test resourceVersion: "14627" uid: 3db464a8-101f-4abb-84dc-c2dd96792242 kind: Event lastTimestamp: "2026-04-16T18:15:44Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1412" in 13.499s (13.499s including waiting). Image size: 1332224104 bytes.' metadata: creationTimestamp: "2026-04-16T18:15:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:15:44Z" name: error-404-isvc-390f9-predictor-666f56949b-vxbx4.18a6e90ce5ce239e namespace: kserve-ci-e2e-test resourceVersion: "14842" uid: 0a926698-7cd1-4b38-a665-6797acad8e9d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:15:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-390f9-predictor-666f56949b-vxbx4 namespace: kserve-ci-e2e-test resourceVersion: "14627" uid: 3db464a8-101f-4abb-84dc-c2dd96792242 kind: Event lastTimestamp: "2026-04-16T18:15:44Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:15:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:15:44Z" name: error-404-isvc-390f9-predictor-666f56949b-vxbx4.18a6e90cea830830 namespace: kserve-ci-e2e-test resourceVersion: "14843" uid: 4e2f4153-ebef-4205-b9ce-50ebd1e6eca9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:15:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-390f9-predictor-666f56949b-vxbx4 namespace: kserve-ci-e2e-test resourceVersion: "14627" uid: 3db464a8-101f-4abb-84dc-c2dd96792242 kind: Event lastTimestamp: "2026-04-16T18:15:44Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:15:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:15:44Z" name: error-404-isvc-390f9-predictor-666f56949b-vxbx4.18a6e90ceb0359ef namespace: kserve-ci-e2e-test resourceVersion: "14844" uid: 9cc48523-75ce-450b-b896-fe0eec1e199e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:15:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-390f9-predictor-666f56949b-vxbx4 namespace: kserve-ci-e2e-test resourceVersion: "14627" uid: 3db464a8-101f-4abb-84dc-c2dd96792242 kind: Event lastTimestamp: "2026-04-16T18:16:26Z" message: 'Readiness probe failed: dial tcp 10.132.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:15:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:16:26Z" name: error-404-isvc-390f9-predictor-666f56949b-vxbx4.18a6e90d0ad78437 namespace: kserve-ci-e2e-test resourceVersion: "15219" uid: 4f3099fe-aa38-4297-82fa-63fd38a935b7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:17:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-390f9-predictor-666f56949b-vxbx4 namespace: kserve-ci-e2e-test resourceVersion: "14627" uid: 3db464a8-101f-4abb-84dc-c2dd96792242 kind: Event lastTimestamp: "2026-04-16T18:17:00Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:17:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:17:00Z" name: error-404-isvc-390f9-predictor-666f56949b-vxbx4.18a6e91e99bde938 namespace: kserve-ci-e2e-test resourceVersion: "15632" uid: 5a62def5-2e42-44c2-b921-bc1eb0e256d0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:15:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-390f9-predictor-666f56949b namespace: kserve-ci-e2e-test resourceVersion: "14615" uid: fd092659-f489-4b7e-a8c7-f479926314bf kind: Event lastTimestamp: "2026-04-16T18:15:30Z" message: 'Created pod: error-404-isvc-390f9-predictor-666f56949b-vxbx4' metadata: creationTimestamp: "2026-04-16T18:15:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:15:30Z" name: error-404-isvc-390f9-predictor-666f56949b.18a6e909a7efb3e7 namespace: kserve-ci-e2e-test resourceVersion: "14630" uid: 64c38c87-a21e-4375-abfe-f0e419e12733 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:15:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-390f9-predictor namespace: kserve-ci-e2e-test resourceVersion: "14612" uid: a2efdd9c-8a76-4ec7-a401-31a9ee40db2a kind: Event lastTimestamp: "2026-04-16T18:15:30Z" message: Scaled up replica set error-404-isvc-390f9-predictor-666f56949b from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:15:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:15:30Z" name: error-404-isvc-390f9-predictor.18a6e909a230f765 namespace: kserve-ci-e2e-test resourceVersion: "14620" uid: 1b19f11a-b6b8-4fbb-851d-d0182ed4ee84 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:15:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-390f9-predictor namespace: kserve-ci-e2e-test resourceVersion: "14614" uid: 43705187-d55b-4179-9c5d-a806d5a5b88c kind: Event lastTimestamp: "2026-04-16T18:15:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:15:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:15:45Z" name: error-404-isvc-390f9-predictor.18a6e90d20b17658 namespace: kserve-ci-e2e-test resourceVersion: "14874" uid: 2ca3c6cb-3fd5-46ad-bdc2-020532174adb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:15:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-390f9-predictor namespace: kserve-ci-e2e-test resourceVersion: "14614" uid: 43705187-d55b-4179-9c5d-a806d5a5b88c kind: Event lastTimestamp: "2026-04-16T18:15:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:15:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:15:45Z" name: error-404-isvc-390f9-predictor.18a6e90d2125a8ec namespace: kserve-ci-e2e-test resourceVersion: "14876" uid: b076a059-1840-44f5-9c63-b7a54056746a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T18:16:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-390f9-predictor namespace: kserve-ci-e2e-test resourceVersion: "14875" uid: 43705187-d55b-4179-9c5d-a806d5a5b88c kind: Event lastTimestamp: "2026-04-16T18:16:45Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:16:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:16:45Z" name: error-404-isvc-390f9-predictor.18a6e9109ff48b04 namespace: kserve-ci-e2e-test resourceVersion: "15390" uid: b7c95f51-73a7-47d8-9db3-4fe6022a432c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T18:16:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-390f9-predictor namespace: kserve-ci-e2e-test resourceVersion: "14875" uid: 43705187-d55b-4179-9c5d-a806d5a5b88c kind: Event lastTimestamp: "2026-04-16T18:16:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:16:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:16:45Z" name: error-404-isvc-390f9-predictor.18a6e910a066083b namespace: kserve-ci-e2e-test resourceVersion: "15392" uid: fb6f2934-fd5a-49c5-b2aa-89d9f88aff99 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T18:15:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-390f9 namespace: kserve-ci-e2e-test resourceVersion: "14596" uid: 30b85289-ca0b-4d69-a2f3-cc5868e8defb kind: Event lastTimestamp: "2026-04-16T18:16:36Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:15:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:16:36Z" name: error-404-isvc-390f9.18a6e909a1fb2c83 namespace: kserve-ci-e2e-test resourceVersion: "15322" uid: ece7dc63-afa5-4c25-bf8d-b157b0576512 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:16:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-390f9 namespace: kserve-ci-e2e-test resourceVersion: "15320" uid: 30b85289-ca0b-4d69-a2f3-cc5868e8defb kind: Event lastTimestamp: "2026-04-16T18:16:36Z" message: InferenceService [error-404-isvc-390f9] is Ready metadata: creationTimestamp: "2026-04-16T18:16:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:16:36Z" name: error-404-isvc-390f9.18a6e918ef46c933 namespace: kserve-ci-e2e-test resourceVersion: "15321" uid: ba890d70-2774-4e0b-972c-0d36bea4d73a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:26:55.652857Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-516af-predictor-5f8c67c956-pwv6b namespace: kserve-ci-e2e-test resourceVersion: "20411" uid: 99bcb2ca-e0ce-4d51-a4d9-428ae55d0989 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-516af-predictor-5f8c67c956-pwv6b to ip-10-0-134-133.ec2.internal metadata: creationTimestamp: "2026-04-16T18:26:55Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:26:55Z" name: error-404-isvc-516af-predictor-5f8c67c956-pwv6b.18a6e9a920c68939 namespace: kserve-ci-e2e-test resourceVersion: "20417" uid: 9b787a38-d010-470e-8555-6450c724ce7c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4b7bbd69-nffhf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:55Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-516af-predictor-5f8c67c956-pwv6b namespace: kserve-ci-e2e-test resourceVersion: "20421" uid: 99bcb2ca-e0ce-4d51-a4d9-428ae55d0989 kind: Event lastTimestamp: "2026-04-16T18:26:55Z" message: Add eth0 [10.132.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:26:55Z" name: error-404-isvc-516af-predictor-5f8c67c956-pwv6b.18a6e9a9293f2c90 namespace: kserve-ci-e2e-test resourceVersion: "20439" uid: 6d3b54bd-8635-45a7-8337-c77afce311fa reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-516af-predictor-5f8c67c956-pwv6b namespace: kserve-ci-e2e-test resourceVersion: "20415" uid: 99bcb2ca-e0ce-4d51-a4d9-428ae55d0989 kind: Event lastTimestamp: "2026-04-16T18:26:55Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1412" already present on machine metadata: creationTimestamp: "2026-04-16T18:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:26:55Z" name: error-404-isvc-516af-predictor-5f8c67c956-pwv6b.18a6e9a92a9b6f54 namespace: kserve-ci-e2e-test resourceVersion: "20442" uid: eb226386-1866-40f8-bc4e-f0484d0aec7c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-516af-predictor-5f8c67c956-pwv6b namespace: kserve-ci-e2e-test resourceVersion: "20415" uid: 99bcb2ca-e0ce-4d51-a4d9-428ae55d0989 kind: Event lastTimestamp: "2026-04-16T18:26:55Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:26:55Z" name: error-404-isvc-516af-predictor-5f8c67c956-pwv6b.18a6e9a93145e174 namespace: kserve-ci-e2e-test resourceVersion: "20452" uid: c94ff532-7dca-46c7-b24f-a33f4f76af60 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-516af-predictor-5f8c67c956-pwv6b namespace: kserve-ci-e2e-test resourceVersion: "20415" uid: 99bcb2ca-e0ce-4d51-a4d9-428ae55d0989 kind: Event lastTimestamp: "2026-04-16T18:26:55Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:26:55Z" name: error-404-isvc-516af-predictor-5f8c67c956-pwv6b.18a6e9a931bdf071 namespace: kserve-ci-e2e-test resourceVersion: "20453" uid: 1b7557ab-2cea-477e-baae-1fc1400dabf6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T18:26:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-516af-predictor-5f8c67c956-pwv6b namespace: kserve-ci-e2e-test resourceVersion: "20415" uid: 99bcb2ca-e0ce-4d51-a4d9-428ae55d0989 kind: Event lastTimestamp: "2026-04-16T18:28:17Z" message: 'Readiness probe failed: dial tcp 10.132.0.43:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:26:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:28:17Z" name: error-404-isvc-516af-predictor-5f8c67c956-pwv6b.18a6e9a95f99aa1d namespace: kserve-ci-e2e-test resourceVersion: "21486" uid: 2b1c9365-8677-4fde-a231-343647bedfbf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-516af-predictor-5f8c67c956-pwv6b namespace: kserve-ci-e2e-test resourceVersion: "20415" uid: 99bcb2ca-e0ce-4d51-a4d9-428ae55d0989 kind: Event lastTimestamp: "2026-04-16T18:28:15Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:28:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:28:15Z" name: error-404-isvc-516af-predictor-5f8c67c956-pwv6b.18a6e9bbcfc00b2c namespace: kserve-ci-e2e-test resourceVersion: "21428" uid: 7f4b4f99-4ae4-492e-8741-62d8053efd9c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:55Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-516af-predictor-5f8c67c956 namespace: kserve-ci-e2e-test resourceVersion: "20403" uid: 5da1007c-abed-4ce1-b6eb-9f23d6b5bfcf kind: Event lastTimestamp: "2026-04-16T18:26:55Z" message: 'Created pod: error-404-isvc-516af-predictor-5f8c67c956-pwv6b' metadata: creationTimestamp: "2026-04-16T18:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:26:55Z" name: error-404-isvc-516af-predictor-5f8c67c956.18a6e9a9204241ae namespace: kserve-ci-e2e-test resourceVersion: "20413" uid: f258f429-4aff-4d53-bd97-d726abfe1ada reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:55Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-516af-predictor namespace: kserve-ci-e2e-test resourceVersion: "20396" uid: 744a0190-d3c8-47e7-ac75-fec4e046c74c kind: Event lastTimestamp: "2026-04-16T18:26:55Z" message: Scaled up replica set error-404-isvc-516af-predictor-5f8c67c956 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:26:55Z" name: error-404-isvc-516af-predictor.18a6e9a91ee05c01 namespace: kserve-ci-e2e-test resourceVersion: "20405" uid: de7a9355-11d5-4887-8630-64dd040be6d6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:27:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-516af-predictor namespace: kserve-ci-e2e-test resourceVersion: "20402" uid: 05528246-9b8b-4b0d-b9fd-281027bb93c6 kind: Event lastTimestamp: "2026-04-16T18:27:25Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:27:25Z" name: error-404-isvc-516af-predictor.18a6e9ac9da1b61e namespace: kserve-ci-e2e-test resourceVersion: "20716" uid: 291c29f1-7a0e-4eb4-b1ed-ab1fd05eff89 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:27:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-516af-predictor namespace: kserve-ci-e2e-test resourceVersion: "20402" uid: 05528246-9b8b-4b0d-b9fd-281027bb93c6 kind: Event lastTimestamp: "2026-04-16T18:27:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:27:25Z" name: error-404-isvc-516af-predictor.18a6e9ac9e0aa5e3 namespace: kserve-ci-e2e-test resourceVersion: "20717" uid: eb754650-8a4d-47d2-a212-04cb3c5b5741 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:27:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-516af-predictor namespace: kserve-ci-e2e-test resourceVersion: "20581" uid: 05528246-9b8b-4b0d-b9fd-281027bb93c6 kind: Event lastTimestamp: "2026-04-16T18:28:10Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:27:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:28:10Z" name: error-404-isvc-516af-predictor.18a6e9b39bc61dc7 namespace: kserve-ci-e2e-test resourceVersion: "21298" uid: dfbac035-bc1d-4965-84e0-54902b04dd12 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:27:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-516af-predictor namespace: kserve-ci-e2e-test resourceVersion: "20581" uid: 05528246-9b8b-4b0d-b9fd-281027bb93c6 kind: Event lastTimestamp: "2026-04-16T18:28:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:27:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:28:10Z" name: error-404-isvc-516af-predictor.18a6e9b39c36f716 namespace: kserve-ci-e2e-test resourceVersion: "21299" uid: 30aa6328-4dce-4ff9-b84a-3fdac15760fb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T18:26:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-516af namespace: kserve-ci-e2e-test resourceVersion: "20385" uid: 678fa4e8-7283-41fc-a6a9-c28ff42fb3d0 kind: Event lastTimestamp: "2026-04-16T18:27:47Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:27:47Z" name: error-404-isvc-516af.18a6e9a91ee638d3 namespace: kserve-ci-e2e-test resourceVersion: "21080" uid: c078eabf-66a4-462d-8c18-d74698e55d17 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-516af namespace: kserve-ci-e2e-test resourceVersion: "20385" uid: 678fa4e8-7283-41fc-a6a9-c28ff42fb3d0 kind: Event lastTimestamp: "2026-04-16T18:26:55Z" message: 'Failed to update status for InferenceService "error-404-isvc-516af": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-516af": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:26:55Z" name: error-404-isvc-516af.18a6e9a91f857537 namespace: kserve-ci-e2e-test resourceVersion: "20410" uid: 2762a024-b061-4d9f-b6e3-460354c63af9 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-516af namespace: kserve-ci-e2e-test resourceVersion: "20385" uid: 678fa4e8-7283-41fc-a6a9-c28ff42fb3d0 kind: Event lastTimestamp: "2026-04-16T18:26:55Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-516af": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:26:55Z" name: error-404-isvc-516af.18a6e9a91f85c870 namespace: kserve-ci-e2e-test resourceVersion: "20412" uid: 6179a1c0-6383-4e54-85d8-05e50fb6a21d reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-516af namespace: kserve-ci-e2e-test resourceVersion: "21078" uid: 678fa4e8-7283-41fc-a6a9-c28ff42fb3d0 kind: Event lastTimestamp: "2026-04-16T18:27:47Z" message: InferenceService [error-404-isvc-516af] is Ready metadata: creationTimestamp: "2026-04-16T18:27:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:27:47Z" name: error-404-isvc-516af.18a6e9b5440ce6b8 namespace: kserve-ci-e2e-test resourceVersion: "21079" uid: ff671dfb-cb14-42f4-b1ac-e174ed60c0b3 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:37:10.989482Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-78581-predictor-8648c458b4-v2c7w namespace: kserve-ci-e2e-test resourceVersion: "25683" uid: 88f23671-8785-4093-9eeb-d0d4a01e3309 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-78581-predictor-8648c458b4-v2c7w to ip-10-0-134-133.ec2.internal metadata: creationTimestamp: "2026-04-16T18:37:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:37:10Z" name: error-404-isvc-78581-predictor-8648c458b4-v2c7w.18a6ea3865b23cf4 namespace: kserve-ci-e2e-test resourceVersion: "25687" uid: cce77a1b-cbe7-4f71-b743-7dd5866b8330 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4b7bbd69-nffhf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:11Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-78581-predictor-8648c458b4-v2c7w namespace: kserve-ci-e2e-test resourceVersion: "25692" uid: 88f23671-8785-4093-9eeb-d0d4a01e3309 kind: Event lastTimestamp: "2026-04-16T18:37:11Z" message: Add eth0 [10.132.0.46/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:37:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:37:11Z" name: error-404-isvc-78581-predictor-8648c458b4-v2c7w.18a6ea386df0e13c namespace: kserve-ci-e2e-test resourceVersion: "25700" uid: 5d9be231-33c7-4ff8-845c-8f20312fecb0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-78581-predictor-8648c458b4-v2c7w namespace: kserve-ci-e2e-test resourceVersion: "25685" uid: 88f23671-8785-4093-9eeb-d0d4a01e3309 kind: Event lastTimestamp: "2026-04-16T18:37:11Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1412" already present on machine metadata: creationTimestamp: "2026-04-16T18:37:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:37:11Z" name: error-404-isvc-78581-predictor-8648c458b4-v2c7w.18a6ea386f1c3dac namespace: kserve-ci-e2e-test resourceVersion: "25703" uid: fd2bae0a-ed99-420d-b49d-79068bc89b2e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-78581-predictor-8648c458b4-v2c7w namespace: kserve-ci-e2e-test resourceVersion: "25685" uid: 88f23671-8785-4093-9eeb-d0d4a01e3309 kind: Event lastTimestamp: "2026-04-16T18:37:11Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:37:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:37:11Z" name: error-404-isvc-78581-predictor-8648c458b4-v2c7w.18a6ea3873f5cf98 namespace: kserve-ci-e2e-test resourceVersion: "25710" uid: f9abbb6d-2c95-4267-8d02-36eb59cc46c9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-78581-predictor-8648c458b4-v2c7w namespace: kserve-ci-e2e-test resourceVersion: "25685" uid: 88f23671-8785-4093-9eeb-d0d4a01e3309 kind: Event lastTimestamp: "2026-04-16T18:37:11Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:37:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:37:11Z" name: error-404-isvc-78581-predictor-8648c458b4-v2c7w.18a6ea387467c0bb namespace: kserve-ci-e2e-test resourceVersion: "25711" uid: 3c1e9dcc-ccc6-4829-acf8-34594115c921 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T18:37:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-78581-predictor-8648c458b4-v2c7w namespace: kserve-ci-e2e-test resourceVersion: "25685" uid: 88f23671-8785-4093-9eeb-d0d4a01e3309 kind: Event lastTimestamp: "2026-04-16T18:38:32Z" message: 'Readiness probe failed: dial tcp 10.132.0.46:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:37:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:38:32Z" name: error-404-isvc-78581-predictor-8648c458b4-v2c7w.18a6ea389a8c6be4 namespace: kserve-ci-e2e-test resourceVersion: "26687" uid: b9d0cac8-c106-47f4-80c2-274d8934d18b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-78581-predictor-8648c458b4-v2c7w namespace: kserve-ci-e2e-test resourceVersion: "25685" uid: 88f23671-8785-4093-9eeb-d0d4a01e3309 kind: Event lastTimestamp: "2026-04-16T18:38:31Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:38:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:38:31Z" name: error-404-isvc-78581-predictor-8648c458b4-v2c7w.18a6ea4b17342d8b namespace: kserve-ci-e2e-test resourceVersion: "26625" uid: f0567c54-160e-4691-9efc-5cd17168c097 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-78581-predictor-8648c458b4 namespace: kserve-ci-e2e-test resourceVersion: "25674" uid: de9199c1-f5e6-40f6-b519-807f8193f5ec kind: Event lastTimestamp: "2026-04-16T18:37:10Z" message: 'Created pod: error-404-isvc-78581-predictor-8648c458b4-v2c7w' metadata: creationTimestamp: "2026-04-16T18:37:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:37:10Z" name: error-404-isvc-78581-predictor-8648c458b4.18a6ea3865224621 namespace: kserve-ci-e2e-test resourceVersion: "25684" uid: 553583c3-03ec-4655-b562-7a2abf1dc739 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-78581-predictor namespace: kserve-ci-e2e-test resourceVersion: "25668" uid: 1e1a84e5-fc5a-4c6a-af16-d25d03b4541a kind: Event lastTimestamp: "2026-04-16T18:37:10Z" message: Scaled up replica set error-404-isvc-78581-predictor-8648c458b4 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:37:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:37:10Z" name: error-404-isvc-78581-predictor.18a6ea3863a1bb65 namespace: kserve-ci-e2e-test resourceVersion: "25677" uid: 8f90a2dc-749a-4af5-9edf-1d7d21a1a683 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:37:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-78581-predictor namespace: kserve-ci-e2e-test resourceVersion: "25673" uid: 4ff3ed00-0f29-437c-bb63-c5aa93684168 kind: Event lastTimestamp: "2026-04-16T18:37:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:37:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:37:56Z" name: error-404-isvc-78581-predictor.18a6ea3be2d86ffb namespace: kserve-ci-e2e-test resourceVersion: "26208" uid: c403ab90-cac2-4906-a47f-5ec08234fa40 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:37:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-78581-predictor namespace: kserve-ci-e2e-test resourceVersion: "25673" uid: 4ff3ed00-0f29-437c-bb63-c5aa93684168 kind: Event lastTimestamp: "2026-04-16T18:37:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:37:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:37:56Z" name: error-404-isvc-78581-predictor.18a6ea3be3913112 namespace: kserve-ci-e2e-test resourceVersion: "26209" uid: af7eb5e0-588d-4fa3-a78e-200a064b1103 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:38:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-78581-predictor namespace: kserve-ci-e2e-test resourceVersion: "25843" uid: 4ff3ed00-0f29-437c-bb63-c5aa93684168 kind: Event lastTimestamp: "2026-04-16T18:38:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:38:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:38:26Z" name: error-404-isvc-78581-predictor.18a6ea465ffe525b namespace: kserve-ci-e2e-test resourceVersion: "26500" uid: 78dc31ca-17eb-4f17-afe2-ee22bb912afa reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:38:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-78581-predictor namespace: kserve-ci-e2e-test resourceVersion: "25843" uid: 4ff3ed00-0f29-437c-bb63-c5aa93684168 kind: Event lastTimestamp: "2026-04-16T18:38:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:38:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:38:26Z" name: error-404-isvc-78581-predictor.18a6ea466069e875 namespace: kserve-ci-e2e-test resourceVersion: "26501" uid: 14f43cea-efab-4dd9-bdad-481a5199d121 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:37:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-78581 namespace: kserve-ci-e2e-test resourceVersion: "25645" uid: b2012449-1bfc-4e85-a683-284d472557bc kind: Event lastTimestamp: "2026-04-16T18:38:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:37:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:38:02Z" name: error-404-isvc-78581.18a6ea3863a9bb87 namespace: kserve-ci-e2e-test resourceVersion: "26276" uid: 30859598-16b2-4f92-a2e1-0f54299555d4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-78581 namespace: kserve-ci-e2e-test resourceVersion: "25645" uid: b2012449-1bfc-4e85-a683-284d472557bc kind: Event lastTimestamp: "2026-04-16T18:37:10Z" message: 'Failed to update status for InferenceService "error-404-isvc-78581": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-78581": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:37:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:37:10Z" name: error-404-isvc-78581.18a6ea3864478a8f namespace: kserve-ci-e2e-test resourceVersion: "25680" uid: 2bacf194-6927-4b9b-a7e4-3e8a6a7e45f0 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-78581 namespace: kserve-ci-e2e-test resourceVersion: "25645" uid: b2012449-1bfc-4e85-a683-284d472557bc kind: Event lastTimestamp: "2026-04-16T18:37:10Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-78581": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:37:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:37:10Z" name: error-404-isvc-78581.18a6ea386447c699 namespace: kserve-ci-e2e-test resourceVersion: "25682" uid: 31667591-9953-45ff-9dbd-4d113bdab3f6 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-78581 namespace: kserve-ci-e2e-test resourceVersion: "26274" uid: b2012449-1bfc-4e85-a683-284d472557bc kind: Event lastTimestamp: "2026-04-16T18:38:02Z" message: InferenceService [error-404-isvc-78581] is Ready metadata: creationTimestamp: "2026-04-16T18:38:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:38:02Z" name: error-404-isvc-78581.18a6ea447ea3196c namespace: kserve-ci-e2e-test resourceVersion: "26275" uid: b221f06c-7912-475c-a52b-a91e3d708884 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:38:31.291224Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-84e06-predictor-59d6c4b5b8-qwgj5 namespace: kserve-ci-e2e-test resourceVersion: "26626" uid: 52716b75-befe-4500-9562-fb6fdc3d46de kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-84e06-predictor-59d6c4b5b8-qwgj5 to ip-10-0-134-133.ec2.internal metadata: creationTimestamp: "2026-04-16T18:38:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:38:31Z" name: error-404-isvc-84e06-predictor-59d6c4b5b8-qwgj5.18a6ea4b180d9cd1 namespace: kserve-ci-e2e-test resourceVersion: "26631" uid: 764ecde4-d875-4960-8a8c-74df11ba18a7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4b7bbd69-nffhf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:31Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-84e06-predictor-59d6c4b5b8-qwgj5 namespace: kserve-ci-e2e-test resourceVersion: "26635" uid: 52716b75-befe-4500-9562-fb6fdc3d46de kind: Event lastTimestamp: "2026-04-16T18:38:31Z" message: Add eth0 [10.132.0.48/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:38:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:38:31Z" name: error-404-isvc-84e06-predictor-59d6c4b5b8-qwgj5.18a6ea4b20e7564d namespace: kserve-ci-e2e-test resourceVersion: "26650" uid: e6bb8e40-cb38-4fda-ba73-9540a5f43672 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-84e06-predictor-59d6c4b5b8-qwgj5 namespace: kserve-ci-e2e-test resourceVersion: "26629" uid: 52716b75-befe-4500-9562-fb6fdc3d46de kind: Event lastTimestamp: "2026-04-16T18:38:31Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1412" already present on machine metadata: creationTimestamp: "2026-04-16T18:38:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:38:31Z" name: error-404-isvc-84e06-predictor-59d6c4b5b8-qwgj5.18a6ea4b2210b498 namespace: kserve-ci-e2e-test resourceVersion: "26652" uid: 5d3c4102-11b7-4b77-9288-bfa30e22afbc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-84e06-predictor-59d6c4b5b8-qwgj5 namespace: kserve-ci-e2e-test resourceVersion: "26629" uid: 52716b75-befe-4500-9562-fb6fdc3d46de kind: Event lastTimestamp: "2026-04-16T18:38:31Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:38:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:38:31Z" name: error-404-isvc-84e06-predictor-59d6c4b5b8-qwgj5.18a6ea4b2649e40f namespace: kserve-ci-e2e-test resourceVersion: "26655" uid: 4a1655e4-fc73-4c4c-b1a1-ff0ad7ceb74f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-84e06-predictor-59d6c4b5b8-qwgj5 namespace: kserve-ci-e2e-test resourceVersion: "26629" uid: 52716b75-befe-4500-9562-fb6fdc3d46de kind: Event lastTimestamp: "2026-04-16T18:38:31Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:38:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:38:31Z" name: error-404-isvc-84e06-predictor-59d6c4b5b8-qwgj5.18a6ea4b26d079a9 namespace: kserve-ci-e2e-test resourceVersion: "26656" uid: 8e806fed-1413-41b2-9da1-080628c4e2e2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:38:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-84e06-predictor-59d6c4b5b8-qwgj5 namespace: kserve-ci-e2e-test resourceVersion: "26629" uid: 52716b75-befe-4500-9562-fb6fdc3d46de kind: Event lastTimestamp: "2026-04-16T18:39:13Z" message: 'Readiness probe failed: dial tcp 10.132.0.48:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:38:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:39:13Z" name: error-404-isvc-84e06-predictor-59d6c4b5b8-qwgj5.18a6ea4b4cf36b2d namespace: kserve-ci-e2e-test resourceVersion: "27063" uid: 0c305c45-dad8-404c-b50b-cd1237d84f19 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:47:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-84e06-predictor-59d6c4b5b8-qwgj5 namespace: kserve-ci-e2e-test resourceVersion: "26629" uid: 52716b75-befe-4500-9562-fb6fdc3d46de kind: Event lastTimestamp: "2026-04-16T18:47:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:47:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:47:56Z" name: error-404-isvc-84e06-predictor-59d6c4b5b8-qwgj5.18a6eace9c88c27f namespace: kserve-ci-e2e-test resourceVersion: "30788" uid: 4d76c520-2d0e-4a4c-9c1f-5181b468da1b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-84e06-predictor-59d6c4b5b8 namespace: kserve-ci-e2e-test resourceVersion: "26612" uid: cae934c2-1bd6-4ad6-8a1a-ef66a31915b8 kind: Event lastTimestamp: "2026-04-16T18:38:31Z" message: 'Created pod: error-404-isvc-84e06-predictor-59d6c4b5b8-qwgj5' metadata: creationTimestamp: "2026-04-16T18:38:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:38:31Z" name: error-404-isvc-84e06-predictor-59d6c4b5b8.18a6ea4b179a8e3c namespace: kserve-ci-e2e-test resourceVersion: "26628" uid: a3594f5a-865e-4f75-911d-9ebe16b7b8f1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-84e06-predictor namespace: kserve-ci-e2e-test resourceVersion: "26608" uid: b94600b9-078a-49e6-8447-fa93b9742516 kind: Event lastTimestamp: "2026-04-16T18:38:31Z" message: Scaled up replica set error-404-isvc-84e06-predictor-59d6c4b5b8 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:38:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:38:31Z" name: error-404-isvc-84e06-predictor.18a6ea4b16181e1c namespace: kserve-ci-e2e-test resourceVersion: "26617" uid: 229c9437-4855-4fa9-8729-57e8e9f62841 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:38:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-84e06-predictor namespace: kserve-ci-e2e-test resourceVersion: "26611" uid: 0e123672-3de1-49cc-bf09-bb3136696613 kind: Event lastTimestamp: "2026-04-16T18:39:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:38:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:39:01Z" name: error-404-isvc-84e06-predictor.18a6ea4e94d54b9b namespace: kserve-ci-e2e-test resourceVersion: "26964" uid: 4549d5cf-1cc4-4440-979d-19c88ddc796d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:38:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-84e06-predictor namespace: kserve-ci-e2e-test resourceVersion: "26611" uid: 0e123672-3de1-49cc-bf09-bb3136696613 kind: Event lastTimestamp: "2026-04-16T18:39:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:38:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:39:01Z" name: error-404-isvc-84e06-predictor.18a6ea4e95927fcb namespace: kserve-ci-e2e-test resourceVersion: "26965" uid: 15824f7b-4a92-4da7-a226-933db335189c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:39:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-84e06-predictor namespace: kserve-ci-e2e-test resourceVersion: "26811" uid: 0e123672-3de1-49cc-bf09-bb3136696613 kind: Event lastTimestamp: "2026-04-16T18:39:31Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:39:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:39:31Z" name: error-404-isvc-84e06-predictor.18a6ea5593205c7a namespace: kserve-ci-e2e-test resourceVersion: "27218" uid: 699f8030-cb37-4cf1-91ab-dd3df201a252 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:39:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-84e06-predictor namespace: kserve-ci-e2e-test resourceVersion: "26811" uid: 0e123672-3de1-49cc-bf09-bb3136696613 kind: Event lastTimestamp: "2026-04-16T18:39:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:39:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:39:31Z" name: error-404-isvc-84e06-predictor.18a6ea5593bb9cb1 namespace: kserve-ci-e2e-test resourceVersion: "27219" uid: 0c8b590e-f7b4-4272-9f94-7c95034c60e2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:38:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-84e06 namespace: kserve-ci-e2e-test resourceVersion: "26593" uid: b29b3cd3-35b4-4e79-b91b-d7a524edddf0 kind: Event lastTimestamp: "2026-04-16T18:39:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:38:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:39:23Z" name: error-404-isvc-84e06.18a6ea4b161f417b namespace: kserve-ci-e2e-test resourceVersion: "27156" uid: 796b5479-f46f-4e75-b30b-f5510af6bd6c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-84e06 namespace: kserve-ci-e2e-test resourceVersion: "26593" uid: b29b3cd3-35b4-4e79-b91b-d7a524edddf0 kind: Event lastTimestamp: "2026-04-16T18:38:31Z" message: 'Failed to update status for InferenceService "error-404-isvc-84e06": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-84e06": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:38:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:38:31Z" name: error-404-isvc-84e06.18a6ea4b16b35c98 namespace: kserve-ci-e2e-test resourceVersion: "26622" uid: f5b64ede-24cd-4420-a123-abf5adbf9f8e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-84e06 namespace: kserve-ci-e2e-test resourceVersion: "26593" uid: b29b3cd3-35b4-4e79-b91b-d7a524edddf0 kind: Event lastTimestamp: "2026-04-16T18:38:31Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-84e06": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:38:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:38:31Z" name: error-404-isvc-84e06.18a6ea4b16b3d2c3 namespace: kserve-ci-e2e-test resourceVersion: "26627" uid: 15a206ba-ab16-4352-b735-0f95697aed86 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-84e06 namespace: kserve-ci-e2e-test resourceVersion: "27154" uid: b29b3cd3-35b4-4e79-b91b-d7a524edddf0 kind: Event lastTimestamp: "2026-04-16T18:39:23Z" message: InferenceService [error-404-isvc-84e06] is Ready metadata: creationTimestamp: "2026-04-16T18:39:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:39:23Z" name: error-404-isvc-84e06.18a6ea57311cd8a0 namespace: kserve-ci-e2e-test resourceVersion: "27155" uid: 28f61b71-8132-4b7b-a6eb-e1ac4992d684 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:37:40.844494Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-89f53-predictor-7b489589b4-4c4pr namespace: kserve-ci-e2e-test resourceVersion: "26031" uid: 889d9c6e-5de9-486e-9c66-ed99adb14530 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-89f53-predictor-7b489589b4-4c4pr to ip-10-0-134-133.ec2.internal metadata: creationTimestamp: "2026-04-16T18:37:40Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:37:40Z" name: error-404-isvc-89f53-predictor-7b489589b4-4c4pr.18a6ea3f59319b41 namespace: kserve-ci-e2e-test resourceVersion: "26036" uid: 90f56038-c5ee-4cac-8907-70d4b7882487 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4b7bbd69-nffhf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:40Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-89f53-predictor-7b489589b4-4c4pr namespace: kserve-ci-e2e-test resourceVersion: "26039" uid: 889d9c6e-5de9-486e-9c66-ed99adb14530 kind: Event lastTimestamp: "2026-04-16T18:37:40Z" message: Add eth0 [10.132.0.47/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:37:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:37:40Z" name: error-404-isvc-89f53-predictor-7b489589b4-4c4pr.18a6ea3f62003bea namespace: kserve-ci-e2e-test resourceVersion: "26053" uid: 140c95fd-7e63-46bf-b764-6a364960f812 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-89f53-predictor-7b489589b4-4c4pr namespace: kserve-ci-e2e-test resourceVersion: "26033" uid: 889d9c6e-5de9-486e-9c66-ed99adb14530 kind: Event lastTimestamp: "2026-04-16T18:37:41Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1412" already present on machine metadata: creationTimestamp: "2026-04-16T18:37:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:37:41Z" name: error-404-isvc-89f53-predictor-7b489589b4-4c4pr.18a6ea3f63339087 namespace: kserve-ci-e2e-test resourceVersion: "26057" uid: d28bd303-871a-46c2-a3ca-066e2ae30c18 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-89f53-predictor-7b489589b4-4c4pr namespace: kserve-ci-e2e-test resourceVersion: "26033" uid: 889d9c6e-5de9-486e-9c66-ed99adb14530 kind: Event lastTimestamp: "2026-04-16T18:37:41Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:37:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:37:41Z" name: error-404-isvc-89f53-predictor-7b489589b4-4c4pr.18a6ea3f6a3d4b01 namespace: kserve-ci-e2e-test resourceVersion: "26065" uid: 473080ad-4365-45fa-bf1e-935b0ac9321c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-89f53-predictor-7b489589b4-4c4pr namespace: kserve-ci-e2e-test resourceVersion: "26033" uid: 889d9c6e-5de9-486e-9c66-ed99adb14530 kind: Event lastTimestamp: "2026-04-16T18:37:41Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:37:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:37:41Z" name: error-404-isvc-89f53-predictor-7b489589b4-4c4pr.18a6ea3f6abba0eb namespace: kserve-ci-e2e-test resourceVersion: "26066" uid: bb9d8a1d-7160-453a-b81d-33fd7fc63911 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T18:37:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-89f53-predictor-7b489589b4-4c4pr namespace: kserve-ci-e2e-test resourceVersion: "26033" uid: 889d9c6e-5de9-486e-9c66-ed99adb14530 kind: Event lastTimestamp: "2026-04-16T18:55:12Z" message: 'Readiness probe failed: dial tcp 10.132.0.47:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:37:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:55:12Z" name: error-404-isvc-89f53-predictor-7b489589b4-4c4pr.18a6ea3f9d698e2a namespace: kserve-ci-e2e-test resourceVersion: "33840" uid: 76b0db16-d97a-4ce4-aa44-9a8495390601 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:55:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-89f53-predictor-7b489589b4-4c4pr namespace: kserve-ci-e2e-test resourceVersion: "26033" uid: 889d9c6e-5de9-486e-9c66-ed99adb14530 kind: Event lastTimestamp: "2026-04-16T18:55:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:55:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:55:10Z" name: error-404-isvc-89f53-predictor-7b489589b4-4c4pr.18a6eb33bb1c9fd9 namespace: kserve-ci-e2e-test resourceVersion: "33788" uid: 0368fcf6-d4cf-445e-85bb-6234f92d62f0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:40Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-89f53-predictor-7b489589b4 namespace: kserve-ci-e2e-test resourceVersion: "26022" uid: a923cc70-60a7-4532-acd3-b8f193d25dda kind: Event lastTimestamp: "2026-04-16T18:37:40Z" message: 'Created pod: error-404-isvc-89f53-predictor-7b489589b4-4c4pr' metadata: creationTimestamp: "2026-04-16T18:37:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:37:40Z" name: error-404-isvc-89f53-predictor-7b489589b4.18a6ea3f58d6c62f namespace: kserve-ci-e2e-test resourceVersion: "26034" uid: 3c29cbdf-89f5-4c1d-a725-f4e7717f5c38 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:40Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-89f53-predictor namespace: kserve-ci-e2e-test resourceVersion: "26017" uid: 361f746a-5fdb-4374-baab-0dcc911d4a10 kind: Event lastTimestamp: "2026-04-16T18:37:40Z" message: Scaled up replica set error-404-isvc-89f53-predictor-7b489589b4 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:37:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:37:40Z" name: error-404-isvc-89f53-predictor.18a6ea3f5764d6cb namespace: kserve-ci-e2e-test resourceVersion: "26024" uid: d5e9145d-e1be-4055-9e1a-0e9edf5c8931 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-89f53-predictor namespace: kserve-ci-e2e-test resourceVersion: "26023" uid: d50f08bd-02f6-4dd9-93f2-01fc850d0784 kind: Event lastTimestamp: "2026-04-16T18:37:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:37:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:37:55Z" name: error-404-isvc-89f53-predictor.18a6ea42d63c8625 namespace: kserve-ci-e2e-test resourceVersion: "26202" uid: a206cc0c-2883-45ca-8364-a9ffe963188f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-89f53-predictor namespace: kserve-ci-e2e-test resourceVersion: "26023" uid: d50f08bd-02f6-4dd9-93f2-01fc850d0784 kind: Event lastTimestamp: "2026-04-16T18:37:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:37:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:37:55Z" name: error-404-isvc-89f53-predictor.18a6ea42d6a1fb1d namespace: kserve-ci-e2e-test resourceVersion: "26204" uid: 9f166bf9-9dae-48fc-afcc-40297a5c1cb7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T18:38:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-89f53-predictor namespace: kserve-ci-e2e-test resourceVersion: "26203" uid: d50f08bd-02f6-4dd9-93f2-01fc850d0784 kind: Event lastTimestamp: "2026-04-16T18:38:55Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:38:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:38:55Z" name: error-404-isvc-89f53-predictor.18a6ea46554f840e namespace: kserve-ci-e2e-test resourceVersion: "26920" uid: b1ae041a-febc-4a9a-980a-47ecc55f407f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T18:38:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-89f53-predictor namespace: kserve-ci-e2e-test resourceVersion: "26203" uid: d50f08bd-02f6-4dd9-93f2-01fc850d0784 kind: Event lastTimestamp: "2026-04-16T18:38:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:38:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:38:55Z" name: error-404-isvc-89f53-predictor.18a6ea4655bc5b1c namespace: kserve-ci-e2e-test resourceVersion: "26921" uid: 26949ec9-bc8d-458b-a4c1-8146ffd0470a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:37:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-89f53 namespace: kserve-ci-e2e-test resourceVersion: "25995" uid: 6f39d1a9-c4be-4c6e-b59f-e0f466a778da kind: Event lastTimestamp: "2026-04-16T18:38:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:37:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:38:33Z" name: error-404-isvc-89f53.18a6ea3f5797fcc9 namespace: kserve-ci-e2e-test resourceVersion: "26700" uid: 62d03cdc-fc90-4494-ae04-c1b182a19528 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-89f53 namespace: kserve-ci-e2e-test resourceVersion: "25995" uid: 6f39d1a9-c4be-4c6e-b59f-e0f466a778da kind: Event lastTimestamp: "2026-04-16T18:37:40Z" message: 'Failed to update status for InferenceService "error-404-isvc-89f53": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-89f53": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:37:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:37:40Z" name: error-404-isvc-89f53.18a6ea3f581660e1 namespace: kserve-ci-e2e-test resourceVersion: "26029" uid: b2948f57-186a-4aa1-893c-00f17de9950a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-89f53 namespace: kserve-ci-e2e-test resourceVersion: "25995" uid: 6f39d1a9-c4be-4c6e-b59f-e0f466a778da kind: Event lastTimestamp: "2026-04-16T18:37:40Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-89f53": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:37:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:37:40Z" name: error-404-isvc-89f53.18a6ea3f5816a73e namespace: kserve-ci-e2e-test resourceVersion: "26032" uid: df730e0a-db82-4bca-86a7-301bc3c011f8 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-89f53 namespace: kserve-ci-e2e-test resourceVersion: "26697" uid: 6f39d1a9-c4be-4c6e-b59f-e0f466a778da kind: Event lastTimestamp: "2026-04-16T18:38:33Z" message: InferenceService [error-404-isvc-89f53] is Ready metadata: creationTimestamp: "2026-04-16T18:38:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:38:33Z" name: error-404-isvc-89f53.18a6ea4b81fb1054 namespace: kserve-ci-e2e-test resourceVersion: "26698" uid: d2ec1348-a439-4478-81d5-ffdacdeefb68 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:26:25.807187Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ccdbb-predictor-c7cfb5d-jr9f2 namespace: kserve-ci-e2e-test resourceVersion: "20058" uid: 362627f4-bd02-4e6f-aa32-4bc5fc2874f8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-ccdbb-predictor-c7cfb5d-jr9f2 to ip-10-0-134-133.ec2.internal metadata: creationTimestamp: "2026-04-16T18:26:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:26:25Z" name: error-404-isvc-ccdbb-predictor-c7cfb5d-jr9f2.18a6e9a22dd5bd73 namespace: kserve-ci-e2e-test resourceVersion: "20065" uid: faead550-f57b-4dae-8af5-dea89a732f72 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4b7bbd69-nffhf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:25Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ccdbb-predictor-c7cfb5d-jr9f2 namespace: kserve-ci-e2e-test resourceVersion: "20071" uid: 362627f4-bd02-4e6f-aa32-4bc5fc2874f8 kind: Event lastTimestamp: "2026-04-16T18:26:25Z" message: Add eth0 [10.132.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:26:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:26:25Z" name: error-404-isvc-ccdbb-predictor-c7cfb5d-jr9f2.18a6e9a235d47784 namespace: kserve-ci-e2e-test resourceVersion: "20080" uid: 06d0202b-5e74-475c-9d15-71682c12261a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ccdbb-predictor-c7cfb5d-jr9f2 namespace: kserve-ci-e2e-test resourceVersion: "20062" uid: 362627f4-bd02-4e6f-aa32-4bc5fc2874f8 kind: Event lastTimestamp: "2026-04-16T18:26:25Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1412" already present on machine metadata: creationTimestamp: "2026-04-16T18:26:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:26:25Z" name: error-404-isvc-ccdbb-predictor-c7cfb5d-jr9f2.18a6e9a2375c17d6 namespace: kserve-ci-e2e-test resourceVersion: "20082" uid: 30e9f20b-1993-4f79-b867-d43efcfd0553 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ccdbb-predictor-c7cfb5d-jr9f2 namespace: kserve-ci-e2e-test resourceVersion: "20062" uid: 362627f4-bd02-4e6f-aa32-4bc5fc2874f8 kind: Event lastTimestamp: "2026-04-16T18:26:26Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:26:26Z" name: error-404-isvc-ccdbb-predictor-c7cfb5d-jr9f2.18a6e9a23bfb6828 namespace: kserve-ci-e2e-test resourceVersion: "20089" uid: 98132d73-db4b-4c0c-82b7-1ee92e187e9f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ccdbb-predictor-c7cfb5d-jr9f2 namespace: kserve-ci-e2e-test resourceVersion: "20062" uid: 362627f4-bd02-4e6f-aa32-4bc5fc2874f8 kind: Event lastTimestamp: "2026-04-16T18:26:26Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:26:26Z" name: error-404-isvc-ccdbb-predictor-c7cfb5d-jr9f2.18a6e9a23c6c519a namespace: kserve-ci-e2e-test resourceVersion: "20090" uid: 3d8fb4fd-8b2f-42a6-a2f4-430b302a8f1e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T18:26:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ccdbb-predictor-c7cfb5d-jr9f2 namespace: kserve-ci-e2e-test resourceVersion: "20062" uid: 362627f4-bd02-4e6f-aa32-4bc5fc2874f8 kind: Event lastTimestamp: "2026-04-16T18:27:47Z" message: 'Readiness probe failed: dial tcp 10.132.0.42:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:27:47Z" name: error-404-isvc-ccdbb-predictor-c7cfb5d-jr9f2.18a6e9a25cf688dd namespace: kserve-ci-e2e-test resourceVersion: "21068" uid: f1738517-0b60-4408-89ae-9431516581d8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ccdbb-predictor-c7cfb5d-jr9f2 namespace: kserve-ci-e2e-test resourceVersion: "20062" uid: 362627f4-bd02-4e6f-aa32-4bc5fc2874f8 kind: Event lastTimestamp: "2026-04-16T18:27:46Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:27:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:27:46Z" name: error-404-isvc-ccdbb-predictor-c7cfb5d-jr9f2.18a6e9b4dfbe109a namespace: kserve-ci-e2e-test resourceVersion: "21012" uid: a784aee9-f1ed-4edb-b567-7bc0180ad591 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-ccdbb-predictor-c7cfb5d namespace: kserve-ci-e2e-test resourceVersion: "20045" uid: 8146fd17-18f1-4b94-8214-7846a178e4aa kind: Event lastTimestamp: "2026-04-16T18:26:25Z" message: 'Created pod: error-404-isvc-ccdbb-predictor-c7cfb5d-jr9f2' metadata: creationTimestamp: "2026-04-16T18:26:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:26:25Z" name: error-404-isvc-ccdbb-predictor-c7cfb5d.18a6e9a22d201930 namespace: kserve-ci-e2e-test resourceVersion: "20061" uid: f5f78869-6de6-4ee9-adb2-184c32775d68 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-ccdbb-predictor namespace: kserve-ci-e2e-test resourceVersion: "20039" uid: 13cce744-28d0-4ad1-9fea-f321ed4fb4b9 kind: Event lastTimestamp: "2026-04-16T18:26:25Z" message: Scaled up replica set error-404-isvc-ccdbb-predictor-c7cfb5d from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:26:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:26:25Z" name: error-404-isvc-ccdbb-predictor.18a6e9a22ac6c256 namespace: kserve-ci-e2e-test resourceVersion: "20051" uid: e480bdd4-e4a3-4804-8725-692686abf426 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:26:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ccdbb-predictor namespace: kserve-ci-e2e-test resourceVersion: "20044" uid: 22a858e4-7c5e-4d90-b578-dab623f47079 kind: Event lastTimestamp: "2026-04-16T18:26:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:26:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:26:55Z" name: error-404-isvc-ccdbb-predictor.18a6e9a5a98099ab namespace: kserve-ci-e2e-test resourceVersion: "20437" uid: 239dab84-97ac-40d0-9722-30a581f6bf00 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:26:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ccdbb-predictor namespace: kserve-ci-e2e-test resourceVersion: "20044" uid: 22a858e4-7c5e-4d90-b578-dab623f47079 kind: Event lastTimestamp: "2026-04-16T18:26:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:26:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:26:55Z" name: error-404-isvc-ccdbb-predictor.18a6e9a5a9ec7e7b namespace: kserve-ci-e2e-test resourceVersion: "20438" uid: 2ca41a7e-afd8-4631-be11-7dbb725b35d6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:27:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ccdbb-predictor namespace: kserve-ci-e2e-test resourceVersion: "20222" uid: 22a858e4-7c5e-4d90-b578-dab623f47079 kind: Event lastTimestamp: "2026-04-16T18:27:40Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:27:40Z" name: error-404-isvc-ccdbb-predictor.18a6e9aca78209a7 namespace: kserve-ci-e2e-test resourceVersion: "20880" uid: 71398cad-a487-4e5b-9392-bfeaaec9430d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:27:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ccdbb-predictor namespace: kserve-ci-e2e-test resourceVersion: "20222" uid: 22a858e4-7c5e-4d90-b578-dab623f47079 kind: Event lastTimestamp: "2026-04-16T18:27:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:27:40Z" name: error-404-isvc-ccdbb-predictor.18a6e9aca7fed467 namespace: kserve-ci-e2e-test resourceVersion: "20881" uid: 6a295d7a-1e10-477b-b608-d2eec39e1ee3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:26:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ccdbb namespace: kserve-ci-e2e-test resourceVersion: "20023" uid: 68cffcc7-2783-4463-8ef1-5b0549444a54 kind: Event lastTimestamp: "2026-04-16T18:27:17Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:26:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:27:17Z" name: error-404-isvc-ccdbb.18a6e9a22ace1880 namespace: kserve-ci-e2e-test resourceVersion: "20644" uid: 33162064-056b-4d26-ba20-db27eb1b3da5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ccdbb namespace: kserve-ci-e2e-test resourceVersion: "20642" uid: 68cffcc7-2783-4463-8ef1-5b0549444a54 kind: Event lastTimestamp: "2026-04-16T18:27:17Z" message: InferenceService [error-404-isvc-ccdbb] is Ready metadata: creationTimestamp: "2026-04-16T18:27:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:27:17Z" name: error-404-isvc-ccdbb.18a6e9ae411ae1c3 namespace: kserve-ci-e2e-test resourceVersion: "20643" uid: 153774f1-1637-4049-ae6f-a4a00aba7ffe reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:27:46.117722Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-df772-predictor-5c6cd9cb6b-6bgvz namespace: kserve-ci-e2e-test resourceVersion: "21011" uid: 7554a6c0-607d-4699-ae6a-3825504b1d58 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-df772-predictor-5c6cd9cb6b-6bgvz to ip-10-0-134-133.ec2.internal metadata: creationTimestamp: "2026-04-16T18:27:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:27:46Z" name: error-404-isvc-df772-predictor-5c6cd9cb6b-6bgvz.18a6e9b4e0b73919 namespace: kserve-ci-e2e-test resourceVersion: "21018" uid: 33f187ce-90a8-4322-8824-e045e4366c38 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4b7bbd69-nffhf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:46Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-df772-predictor-5c6cd9cb6b-6bgvz namespace: kserve-ci-e2e-test resourceVersion: "21023" uid: 7554a6c0-607d-4699-ae6a-3825504b1d58 kind: Event lastTimestamp: "2026-04-16T18:27:46Z" message: Add eth0 [10.132.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:27:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:27:46Z" name: error-404-isvc-df772-predictor-5c6cd9cb6b-6bgvz.18a6e9b4e85e8259 namespace: kserve-ci-e2e-test resourceVersion: "21032" uid: 7fc28149-b487-44e6-a556-bd93ca999bdc reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-df772-predictor-5c6cd9cb6b-6bgvz namespace: kserve-ci-e2e-test resourceVersion: "21015" uid: 7554a6c0-607d-4699-ae6a-3825504b1d58 kind: Event lastTimestamp: "2026-04-16T18:27:46Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1412" already present on machine metadata: creationTimestamp: "2026-04-16T18:27:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:27:46Z" name: error-404-isvc-df772-predictor-5c6cd9cb6b-6bgvz.18a6e9b4e99fe239 namespace: kserve-ci-e2e-test resourceVersion: "21035" uid: aa1c99d1-d013-4600-92f2-d8cda8595fa5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-df772-predictor-5c6cd9cb6b-6bgvz namespace: kserve-ci-e2e-test resourceVersion: "21015" uid: 7554a6c0-607d-4699-ae6a-3825504b1d58 kind: Event lastTimestamp: "2026-04-16T18:27:46Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:27:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:27:46Z" name: error-404-isvc-df772-predictor-5c6cd9cb6b-6bgvz.18a6e9b4ed850310 namespace: kserve-ci-e2e-test resourceVersion: "21040" uid: 96029dd6-56aa-46be-9597-ddfbb7ccbdc1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-df772-predictor-5c6cd9cb6b-6bgvz namespace: kserve-ci-e2e-test resourceVersion: "21015" uid: 7554a6c0-607d-4699-ae6a-3825504b1d58 kind: Event lastTimestamp: "2026-04-16T18:27:46Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:27:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:27:46Z" name: error-404-isvc-df772-predictor-5c6cd9cb6b-6bgvz.18a6e9b4ee1e5211 namespace: kserve-ci-e2e-test resourceVersion: "21041" uid: 2a83ebfa-a22d-4edb-b43a-b1b48cfa994b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:27:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-df772-predictor-5c6cd9cb6b-6bgvz namespace: kserve-ci-e2e-test resourceVersion: "21015" uid: 7554a6c0-607d-4699-ae6a-3825504b1d58 kind: Event lastTimestamp: "2026-04-16T18:28:27Z" message: 'Readiness probe failed: dial tcp 10.132.0.44:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:27:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:28:27Z" name: error-404-isvc-df772-predictor-5c6cd9cb6b-6bgvz.18a6e9b50ed61647 namespace: kserve-ci-e2e-test resourceVersion: "21569" uid: 37ae20d4-ec17-491a-b51d-92901ed90674 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-df772-predictor-5c6cd9cb6b-6bgvz namespace: kserve-ci-e2e-test resourceVersion: "21015" uid: 7554a6c0-607d-4699-ae6a-3825504b1d58 kind: Event lastTimestamp: "2026-04-16T18:37:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:37:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:37:10Z" name: error-404-isvc-df772-predictor-5c6cd9cb6b-6bgvz.18a6ea38639be023 namespace: kserve-ci-e2e-test resourceVersion: "25676" uid: 7cdb1125-cdeb-47d3-823f-10b49114958a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:46Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-df772-predictor-5c6cd9cb6b namespace: kserve-ci-e2e-test resourceVersion: "20998" uid: b1a33fcc-3497-4096-a3bf-702498aaa5cd kind: Event lastTimestamp: "2026-04-16T18:27:46Z" message: 'Created pod: error-404-isvc-df772-predictor-5c6cd9cb6b-6bgvz' metadata: creationTimestamp: "2026-04-16T18:27:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:27:46Z" name: error-404-isvc-df772-predictor-5c6cd9cb6b.18a6e9b4e043757a namespace: kserve-ci-e2e-test resourceVersion: "21014" uid: a7ccb616-c5ca-4aed-980a-77511e667406 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:46Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-df772-predictor namespace: kserve-ci-e2e-test resourceVersion: "20993" uid: 42d44bb0-690b-4aca-a744-02d8faa39265 kind: Event lastTimestamp: "2026-04-16T18:27:46Z" message: Scaled up replica set error-404-isvc-df772-predictor-5c6cd9cb6b from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:27:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:27:46Z" name: error-404-isvc-df772-predictor.18a6e9b4de7a13cf namespace: kserve-ci-e2e-test resourceVersion: "21004" uid: 0271a488-efff-4ce0-a56c-2bb5c0ddf01f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:28:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-df772-predictor namespace: kserve-ci-e2e-test resourceVersion: "20997" uid: a31e26ad-7859-4754-a32f-169c41430f0d kind: Event lastTimestamp: "2026-04-16T18:28:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:28:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:28:31Z" name: error-404-isvc-df772-predictor.18a6e9b85d011433 namespace: kserve-ci-e2e-test resourceVersion: "21603" uid: 5be767ff-6c82-4130-9936-06c6255a6292 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:28:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-df772-predictor namespace: kserve-ci-e2e-test resourceVersion: "20997" uid: a31e26ad-7859-4754-a32f-169c41430f0d kind: Event lastTimestamp: "2026-04-16T18:28:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:28:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:28:31Z" name: error-404-isvc-df772-predictor.18a6e9b85d76efc8 namespace: kserve-ci-e2e-test resourceVersion: "21604" uid: 3c38855f-9530-4560-9431-5b5430053dd1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-df772-predictor namespace: kserve-ci-e2e-test resourceVersion: "21192" uid: a31e26ad-7859-4754-a32f-169c41430f0d kind: Event lastTimestamp: "2026-04-16T18:28:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:28:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:28:46Z" name: error-404-isvc-df772-predictor.18a6e9c2da65c6ca namespace: kserve-ci-e2e-test resourceVersion: "21742" uid: 71c951f5-eaa9-4ba8-b6b1-54cd9bbc72ce reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-df772-predictor namespace: kserve-ci-e2e-test resourceVersion: "21192" uid: a31e26ad-7859-4754-a32f-169c41430f0d kind: Event lastTimestamp: "2026-04-16T18:28:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:28:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:28:46Z" name: error-404-isvc-df772-predictor.18a6e9c2dab6ff52 namespace: kserve-ci-e2e-test resourceVersion: "21744" uid: 28105e93-611e-416f-b29a-d105087a96e0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:27:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-df772 namespace: kserve-ci-e2e-test resourceVersion: "20977" uid: 694c9ac0-d214-49b5-90aa-2e7d8662b540 kind: Event lastTimestamp: "2026-04-16T18:28:38Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:27:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:28:38Z" name: error-404-isvc-df772.18a6e9b4de58be75 namespace: kserve-ci-e2e-test resourceVersion: "21662" uid: b4e4b3e0-518c-4e58-8432-aaf04fa3504c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-df772 namespace: kserve-ci-e2e-test resourceVersion: "20977" uid: 694c9ac0-d214-49b5-90aa-2e7d8662b540 kind: Event lastTimestamp: "2026-04-16T18:27:46Z" message: 'Failed to update status for InferenceService "error-404-isvc-df772": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-df772": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:27:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:27:46Z" name: error-404-isvc-df772.18a6e9b4df4ac26f namespace: kserve-ci-e2e-test resourceVersion: "21009" uid: bdfe0a14-709c-4691-9586-9cd24f9db740 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-df772 namespace: kserve-ci-e2e-test resourceVersion: "20977" uid: 694c9ac0-d214-49b5-90aa-2e7d8662b540 kind: Event lastTimestamp: "2026-04-16T18:27:46Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-df772": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:27:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:27:46Z" name: error-404-isvc-df772.18a6e9b4df4b43f7 namespace: kserve-ci-e2e-test resourceVersion: "21013" uid: 66000487-dc0e-44f5-8141-2abb5f3a722e reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-df772 namespace: kserve-ci-e2e-test resourceVersion: "21658" uid: 694c9ac0-d214-49b5-90aa-2e7d8662b540 kind: Event lastTimestamp: "2026-04-16T18:28:37Z" message: InferenceService [error-404-isvc-df772] is Ready metadata: creationTimestamp: "2026-04-16T18:28:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:28:37Z" name: error-404-isvc-df772.18a6e9c0f38aa164 namespace: kserve-ci-e2e-test resourceVersion: "21659" uid: 88406308-f681-49b3-8994-b201583b89e3 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:28:15.938150Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ed24b-predictor-945b44ddd-lpdjg namespace: kserve-ci-e2e-test resourceVersion: "21431" uid: 4e2600d2-6eaa-4f3d-a32a-5df7538e4591 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-ed24b-predictor-945b44ddd-lpdjg to ip-10-0-134-133.ec2.internal metadata: creationTimestamp: "2026-04-16T18:28:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:28:15Z" name: error-404-isvc-ed24b-predictor-945b44ddd-lpdjg.18a6e9bbd226e7cb namespace: kserve-ci-e2e-test resourceVersion: "21438" uid: e86b7b58-9fca-4955-931e-60c037d28735 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4b7bbd69-nffhf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:16Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ed24b-predictor-945b44ddd-lpdjg namespace: kserve-ci-e2e-test resourceVersion: "21442" uid: 4e2600d2-6eaa-4f3d-a32a-5df7538e4591 kind: Event lastTimestamp: "2026-04-16T18:28:16Z" message: Add eth0 [10.132.0.45/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:28:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:28:16Z" name: error-404-isvc-ed24b-predictor-945b44ddd-lpdjg.18a6e9bbda05f96e namespace: kserve-ci-e2e-test resourceVersion: "21451" uid: 633173f7-600e-4e05-a4c5-7e4caa1c4ac6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ed24b-predictor-945b44ddd-lpdjg namespace: kserve-ci-e2e-test resourceVersion: "21436" uid: 4e2600d2-6eaa-4f3d-a32a-5df7538e4591 kind: Event lastTimestamp: "2026-04-16T18:28:16Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1412" already present on machine metadata: creationTimestamp: "2026-04-16T18:28:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:28:16Z" name: error-404-isvc-ed24b-predictor-945b44ddd-lpdjg.18a6e9bbdc484903 namespace: kserve-ci-e2e-test resourceVersion: "21456" uid: 553f6130-598f-4f4a-867f-c99f6124ee4d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ed24b-predictor-945b44ddd-lpdjg namespace: kserve-ci-e2e-test resourceVersion: "21436" uid: 4e2600d2-6eaa-4f3d-a32a-5df7538e4591 kind: Event lastTimestamp: "2026-04-16T18:28:16Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:28:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:28:16Z" name: error-404-isvc-ed24b-predictor-945b44ddd-lpdjg.18a6e9bbe366b733 namespace: kserve-ci-e2e-test resourceVersion: "21465" uid: 2afe90c6-df7a-4161-aa5c-204521b0c65b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ed24b-predictor-945b44ddd-lpdjg namespace: kserve-ci-e2e-test resourceVersion: "21436" uid: 4e2600d2-6eaa-4f3d-a32a-5df7538e4591 kind: Event lastTimestamp: "2026-04-16T18:28:16Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:28:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:28:16Z" name: error-404-isvc-ed24b-predictor-945b44ddd-lpdjg.18a6e9bbe3e37696 namespace: kserve-ci-e2e-test resourceVersion: "21466" uid: e73c7324-207f-4e17-a7ab-e4f68c3ce21e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:28:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ed24b-predictor-945b44ddd-lpdjg namespace: kserve-ci-e2e-test resourceVersion: "21436" uid: 4e2600d2-6eaa-4f3d-a32a-5df7538e4591 kind: Event lastTimestamp: "2026-04-16T18:28:58Z" message: 'Readiness probe failed: dial tcp 10.132.0.45:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:28:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:28:58Z" name: error-404-isvc-ed24b-predictor-945b44ddd-lpdjg.18a6e9bc118bbd4f namespace: kserve-ci-e2e-test resourceVersion: "21862" uid: 4e1e843b-7866-4d6d-ba2c-8dc1591e2c40 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ed24b-predictor-945b44ddd-lpdjg namespace: kserve-ci-e2e-test resourceVersion: "21436" uid: 4e2600d2-6eaa-4f3d-a32a-5df7538e4591 kind: Event lastTimestamp: "2026-04-16T18:37:40Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:37:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:37:40Z" name: error-404-isvc-ed24b-predictor-945b44ddd-lpdjg.18a6ea3f5842506b namespace: kserve-ci-e2e-test resourceVersion: "26030" uid: fec5b896-befa-477c-80f9-ba4e0bdfbe63 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-ed24b-predictor-945b44ddd namespace: kserve-ci-e2e-test resourceVersion: "21422" uid: e566e53e-fbc7-4cba-ba38-3767acaaf2ac kind: Event lastTimestamp: "2026-04-16T18:28:15Z" message: 'Created pod: error-404-isvc-ed24b-predictor-945b44ddd-lpdjg' metadata: creationTimestamp: "2026-04-16T18:28:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:28:15Z" name: error-404-isvc-ed24b-predictor-945b44ddd.18a6e9bbd1b0fb7c namespace: kserve-ci-e2e-test resourceVersion: "21435" uid: a5284336-e342-45ef-8472-971d00e4bc70 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-ed24b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21418" uid: 178f09a1-bd1d-462e-b66e-7395b08db7d4 kind: Event lastTimestamp: "2026-04-16T18:28:15Z" message: Scaled up replica set error-404-isvc-ed24b-predictor-945b44ddd from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:28:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:28:15Z" name: error-404-isvc-ed24b-predictor.18a6e9bbcf927bf1 namespace: kserve-ci-e2e-test resourceVersion: "21425" uid: 5dcd730e-7437-4a9e-a3b2-a015f6452612 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:28:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ed24b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21423" uid: 73d1d97d-c61a-49e2-b559-25d5e8a42dc5 kind: Event lastTimestamp: "2026-04-16T18:28:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:28:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:28:45Z" name: error-404-isvc-ed24b-predictor.18a6e9bf4e46bc68 namespace: kserve-ci-e2e-test resourceVersion: "21736" uid: 4a0f8c32-7993-484c-8eb7-554d6916db26 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:28:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ed24b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21423" uid: 73d1d97d-c61a-49e2-b559-25d5e8a42dc5 kind: Event lastTimestamp: "2026-04-16T18:28:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:28:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:28:45Z" name: error-404-isvc-ed24b-predictor.18a6e9bf4eb2904e namespace: kserve-ci-e2e-test resourceVersion: "21737" uid: 9af9333c-f3aa-4050-8be6-3d6489c4ba4b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:29:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ed24b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21598" uid: 73d1d97d-c61a-49e2-b559-25d5e8a42dc5 kind: Event lastTimestamp: "2026-04-16T18:29:30Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:29:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:29:30Z" name: error-404-isvc-ed24b-predictor.18a6e9c64c42fa7d namespace: kserve-ci-e2e-test resourceVersion: "22171" uid: f3d8d4f2-2f2d-4f34-945b-d9b4d5361b4d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:29:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ed24b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21598" uid: 73d1d97d-c61a-49e2-b559-25d5e8a42dc5 kind: Event lastTimestamp: "2026-04-16T18:29:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:29:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:29:30Z" name: error-404-isvc-ed24b-predictor.18a6e9c64cbb753e namespace: kserve-ci-e2e-test resourceVersion: "22172" uid: e0687f43-d3aa-48e4-90ee-85cc92416c71 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T18:28:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ed24b namespace: kserve-ci-e2e-test resourceVersion: "21396" uid: c3bf9183-5307-4cc6-baf5-b1acbdb5f90c kind: Event lastTimestamp: "2026-04-16T18:29:08Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:28:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:29:08Z" name: error-404-isvc-ed24b.18a6e9bbcf9edf9e namespace: kserve-ci-e2e-test resourceVersion: "21954" uid: 1c673b99-2255-4466-af56-677a3615ca9d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ed24b namespace: kserve-ci-e2e-test resourceVersion: "21396" uid: c3bf9183-5307-4cc6-baf5-b1acbdb5f90c kind: Event lastTimestamp: "2026-04-16T18:28:15Z" message: 'Failed to update status for InferenceService "error-404-isvc-ed24b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-ed24b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:28:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:28:15Z" name: error-404-isvc-ed24b.18a6e9bbd094e76a namespace: kserve-ci-e2e-test resourceVersion: "21430" uid: 6c673f9b-54a2-4b81-b0c6-f97cd3bf2bbd reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ed24b namespace: kserve-ci-e2e-test resourceVersion: "21396" uid: c3bf9183-5307-4cc6-baf5-b1acbdb5f90c kind: Event lastTimestamp: "2026-04-16T18:28:15Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-ed24b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:28:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:28:15Z" name: error-404-isvc-ed24b.18a6e9bbd09542a6 namespace: kserve-ci-e2e-test resourceVersion: "21434" uid: b3ea80ee-785a-4487-8886-d8952a06c00b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:29:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ed24b namespace: kserve-ci-e2e-test resourceVersion: "21952" uid: c3bf9183-5307-4cc6-baf5-b1acbdb5f90c kind: Event lastTimestamp: "2026-04-16T18:29:08Z" message: InferenceService [error-404-isvc-ed24b] is Ready metadata: creationTimestamp: "2026-04-16T18:29:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:29:08Z" name: error-404-isvc-ed24b.18a6e9c7f5777566 namespace: kserve-ci-e2e-test resourceVersion: "21953" uid: 050d412d-c536-47a0-adee-b3c66f826bbf reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:15:30.574765Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-74d68d6bdc-xqs95 namespace: kserve-ci-e2e-test resourceVersion: "14601" uid: c07179db-ff3e-446a-be1f-c11a701a6f98 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-74d68d6bdc-xqs95 to ip-10-0-134-55.ec2.internal metadata: creationTimestamp: "2026-04-16T18:15:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:15:30Z" name: isvc-sklearn-graph-1-predictor-74d68d6bdc-xqs95.18a6e9099ef06f3b namespace: kserve-ci-e2e-test resourceVersion: "14606" uid: 5d144174-4e5b-4f57-9618-9c7cdd242b46 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4b7bbd69-nffhf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:15:30Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-74d68d6bdc-xqs95 namespace: kserve-ci-e2e-test resourceVersion: "14616" uid: c07179db-ff3e-446a-be1f-c11a701a6f98 kind: Event lastTimestamp: "2026-04-16T18:15:30Z" message: Add eth0 [10.133.0.16/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:15:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:15:31Z" name: isvc-sklearn-graph-1-predictor-74d68d6bdc-xqs95.18a6e909b7c4108e namespace: kserve-ci-e2e-test resourceVersion: "14652" uid: 5a176c8e-0243-47fd-9ec4-2136dffb120b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:15:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-74d68d6bdc-xqs95 namespace: kserve-ci-e2e-test resourceVersion: "14603" uid: c07179db-ff3e-446a-be1f-c11a701a6f98 kind: Event lastTimestamp: "2026-04-16T18:15:31Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:b1408675ea4fdca559a0643f83f1f99c90fa28691ae069983f57f521927df688" metadata: creationTimestamp: "2026-04-16T18:15:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:15:31Z" name: isvc-sklearn-graph-1-predictor-74d68d6bdc-xqs95.18a6e909bc48ca13 namespace: kserve-ci-e2e-test resourceVersion: "14655" uid: 001a8fa9-8926-48f9-9fad-e34d34f1fe56 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-55.ec2.internal source: component: kubelet host: ip-10-0-134-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:15:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-74d68d6bdc-xqs95 namespace: kserve-ci-e2e-test resourceVersion: "14603" uid: c07179db-ff3e-446a-be1f-c11a701a6f98 kind: Event lastTimestamp: "2026-04-16T18:15:34Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:b1408675ea4fdca559a0643f83f1f99c90fa28691ae069983f57f521927df688" in 3.874s (3.874s including waiting). Image size: 299671472 bytes.' metadata: creationTimestamp: "2026-04-16T18:15:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:15:34Z" name: isvc-sklearn-graph-1-predictor-74d68d6bdc-xqs95.18a6e90aa335405d namespace: kserve-ci-e2e-test resourceVersion: "14742" uid: e2169bb8-f89c-420a-9c5f-ee51f60fecbf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-55.ec2.internal source: component: kubelet host: ip-10-0-134-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:15:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-74d68d6bdc-xqs95 namespace: kserve-ci-e2e-test resourceVersion: "14603" uid: c07179db-ff3e-446a-be1f-c11a701a6f98 kind: Event lastTimestamp: "2026-04-16T18:15:34Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T18:15:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:15:34Z" name: isvc-sklearn-graph-1-predictor-74d68d6bdc-xqs95.18a6e90aa5e21dbd namespace: kserve-ci-e2e-test resourceVersion: "14743" uid: 53c748c7-060e-4767-86d5-35f50a65fff3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-55.ec2.internal source: component: kubelet host: ip-10-0-134-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:15:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-74d68d6bdc-xqs95 namespace: kserve-ci-e2e-test resourceVersion: "14603" uid: c07179db-ff3e-446a-be1f-c11a701a6f98 kind: Event lastTimestamp: "2026-04-16T18:15:34Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T18:15:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:15:34Z" name: isvc-sklearn-graph-1-predictor-74d68d6bdc-xqs95.18a6e90aa650107e namespace: kserve-ci-e2e-test resourceVersion: "14744" uid: 9bdae65d-8b69-4605-b0f6-7e26eaa38db1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-55.ec2.internal source: component: kubelet host: ip-10-0-134-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:15:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-74d68d6bdc-xqs95 namespace: kserve-ci-e2e-test resourceVersion: "14603" uid: c07179db-ff3e-446a-be1f-c11a701a6f98 kind: Event lastTimestamp: "2026-04-16T18:15:39Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1412" metadata: creationTimestamp: "2026-04-16T18:15:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:15:39Z" name: isvc-sklearn-graph-1-predictor-74d68d6bdc-xqs95.18a6e90bb006136a namespace: kserve-ci-e2e-test resourceVersion: "14789" uid: 641b625c-94f0-47ea-bb22-2042eb047184 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-55.ec2.internal source: component: kubelet host: ip-10-0-134-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:15:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-74d68d6bdc-xqs95 namespace: kserve-ci-e2e-test resourceVersion: "14603" uid: c07179db-ff3e-446a-be1f-c11a701a6f98 kind: Event lastTimestamp: "2026-04-16T18:15:52Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1412" in 13.13s (13.13s including waiting). Image size: 1560610740 bytes.' metadata: creationTimestamp: "2026-04-16T18:15:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:15:52Z" name: isvc-sklearn-graph-1-predictor-74d68d6bdc-xqs95.18a6e90ebea30355 namespace: kserve-ci-e2e-test resourceVersion: "14940" uid: 0cecbcf4-2b64-464a-a21b-241af38474f6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-55.ec2.internal source: component: kubelet host: ip-10-0-134-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:15:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-74d68d6bdc-xqs95 namespace: kserve-ci-e2e-test resourceVersion: "14603" uid: c07179db-ff3e-446a-be1f-c11a701a6f98 kind: Event lastTimestamp: "2026-04-16T18:15:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:15:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:15:52Z" name: isvc-sklearn-graph-1-predictor-74d68d6bdc-xqs95.18a6e90ec1aa7ab7 namespace: kserve-ci-e2e-test resourceVersion: "14941" uid: 488b9e84-acbc-4f60-aaa3-984cf82ff496 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-55.ec2.internal source: component: kubelet host: ip-10-0-134-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:15:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-74d68d6bdc-xqs95 namespace: kserve-ci-e2e-test resourceVersion: "14603" uid: c07179db-ff3e-446a-be1f-c11a701a6f98 kind: Event lastTimestamp: "2026-04-16T18:15:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:15:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:15:52Z" name: isvc-sklearn-graph-1-predictor-74d68d6bdc-xqs95.18a6e90ec21dab39 namespace: kserve-ci-e2e-test resourceVersion: "14942" uid: 3c28c61f-74e4-410f-8f0b-e5659e36d7da reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-55.ec2.internal source: component: kubelet host: ip-10-0-134-55.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-16T18:15:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-74d68d6bdc-xqs95 namespace: kserve-ci-e2e-test resourceVersion: "14603" uid: c07179db-ff3e-446a-be1f-c11a701a6f98 kind: Event lastTimestamp: "2026-04-16T18:16:54Z" message: 'Readiness probe failed: dial tcp 10.133.0.16:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:15:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:16:54Z" name: isvc-sklearn-graph-1-predictor-74d68d6bdc-xqs95.18a6e90ef568e5c3 namespace: kserve-ci-e2e-test resourceVersion: "15496" uid: a8e5bcb1-7aad-4c5a-9c81-80d7ace50246 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-55.ec2.internal source: component: kubelet host: ip-10-0-134-55.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:17:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-74d68d6bdc-xqs95 namespace: kserve-ci-e2e-test resourceVersion: "14603" uid: c07179db-ff3e-446a-be1f-c11a701a6f98 kind: Event lastTimestamp: "2026-04-16T18:17:30Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:17:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:17:30Z" name: isvc-sklearn-graph-1-predictor-74d68d6bdc-xqs95.18a6e92590f95953 namespace: kserve-ci-e2e-test resourceVersion: "16054" uid: 6406adf5-a309-4261-ace4-57a13863db43 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-55.ec2.internal source: component: kubelet host: ip-10-0-134-55.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:15:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-74d68d6bdc namespace: kserve-ci-e2e-test resourceVersion: "14590" uid: 6ed5c947-bc54-4c78-b844-d7d0176b2821 kind: Event lastTimestamp: "2026-04-16T18:15:30Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-74d68d6bdc-xqs95' metadata: creationTimestamp: "2026-04-16T18:15:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:15:30Z" name: isvc-sklearn-graph-1-predictor-74d68d6bdc.18a6e9099e589f14 namespace: kserve-ci-e2e-test resourceVersion: "14602" uid: c9cd8d38-393a-4708-892d-13530153470a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:15:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14588" uid: 9050b8d4-0ebc-4d98-942d-302e21d927c1 kind: Event lastTimestamp: "2026-04-16T18:15:30Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-74d68d6bdc from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:15:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:15:30Z" name: isvc-sklearn-graph-1-predictor.18a6e909989f633e namespace: kserve-ci-e2e-test resourceVersion: "14592" uid: a0d80643-dce8-46a3-baa0-16a18dbdd6c5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:15:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14589" uid: 8c88c6bf-dc2e-460a-887c-413039e33d56 kind: Event lastTimestamp: "2026-04-16T18:16:15Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:15:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:16:15Z" name: isvc-sklearn-graph-1-predictor.18a6e90d176e0a0d namespace: kserve-ci-e2e-test resourceVersion: "15133" uid: 1652dba9-17e6-428f-acdf-b0c0acb0018a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:15:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14589" uid: 8c88c6bf-dc2e-460a-887c-413039e33d56 kind: Event lastTimestamp: "2026-04-16T18:16:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:15:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:16:15Z" name: isvc-sklearn-graph-1-predictor.18a6e90d17e33e10 namespace: kserve-ci-e2e-test resourceVersion: "15134" uid: aa6455f1-89d0-4020-b027-83da6eb0a5f8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T18:16:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14871" uid: 8c88c6bf-dc2e-460a-887c-413039e33d56 kind: Event lastTimestamp: "2026-04-16T18:17:15Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:16:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:17:15Z" name: isvc-sklearn-graph-1-predictor.18a6e91794583635 namespace: kserve-ci-e2e-test resourceVersion: "15825" uid: 60d4c5c5-32ec-482a-9956-8be115b461db reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T18:16:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14871" uid: 8c88c6bf-dc2e-460a-887c-413039e33d56 kind: Event lastTimestamp: "2026-04-16T18:17:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:16:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:17:15Z" name: isvc-sklearn-graph-1-predictor.18a6e91794ce1de1 namespace: kserve-ci-e2e-test resourceVersion: "15826" uid: 6294daae-779b-48ce-b415-d0180084444e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-16T18:15:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14571" uid: 4184801e-5a77-4ba9-b3e9-e5c4184bae6a kind: Event lastTimestamp: "2026-04-16T18:16:59Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:15:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:16:59Z" name: isvc-sklearn-graph-1.18a6e90998b686e4 namespace: kserve-ci-e2e-test resourceVersion: "15542" uid: 5dbddc75-d516-47dd-af66-7b4dc8b3ab2f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:16:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "15539" uid: 4184801e-5a77-4ba9-b3e9-e5c4184bae6a kind: Event lastTimestamp: "2026-04-16T18:16:59Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-04-16T18:16:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:16:59Z" name: isvc-sklearn-graph-1.18a6e91e48b8926a namespace: kserve-ci-e2e-test resourceVersion: "15540" uid: ab2b45dc-dc1b-485d-9e7e-02fa1a6435fa reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:15:31.254086Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-d76b874f9-gp6ts namespace: kserve-ci-e2e-test resourceVersion: "14685" uid: 6e7fd811-a17a-4ab8-8c18-e26dda756487 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-d76b874f9-gp6ts to ip-10-0-134-133.ec2.internal metadata: creationTimestamp: "2026-04-16T18:15:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:15:31Z" name: isvc-sklearn-graph-2-predictor-d76b874f9-gp6ts.18a6e909c76e0e8b namespace: kserve-ci-e2e-test resourceVersion: "14689" uid: 3c2a75b7-85d6-4f63-bdd9-9b4a7d75a5d7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4b7bbd69-nffhf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:15:31Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-d76b874f9-gp6ts namespace: kserve-ci-e2e-test resourceVersion: "14697" uid: 6e7fd811-a17a-4ab8-8c18-e26dda756487 kind: Event lastTimestamp: "2026-04-16T18:15:31Z" message: Add eth0 [10.132.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:15:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:15:31Z" name: isvc-sklearn-graph-2-predictor-d76b874f9-gp6ts.18a6e909e10542b9 namespace: kserve-ci-e2e-test resourceVersion: "14710" uid: 7797374f-03d6-4719-90f6-cbc81ecb7e20 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:15:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-d76b874f9-gp6ts namespace: kserve-ci-e2e-test resourceVersion: "14686" uid: 6e7fd811-a17a-4ab8-8c18-e26dda756487 kind: Event lastTimestamp: "2026-04-16T18:15:31Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:b1408675ea4fdca559a0643f83f1f99c90fa28691ae069983f57f521927df688" metadata: creationTimestamp: "2026-04-16T18:15:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:15:31Z" name: isvc-sklearn-graph-2-predictor-d76b874f9-gp6ts.18a6e909e36f814d namespace: kserve-ci-e2e-test resourceVersion: "14712" uid: 94c5a80d-312c-478c-bc04-51deeb1363c7 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:15:42Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-d76b874f9-gp6ts namespace: kserve-ci-e2e-test resourceVersion: "14686" uid: 6e7fd811-a17a-4ab8-8c18-e26dda756487 kind: Event lastTimestamp: "2026-04-16T18:15:42Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:b1408675ea4fdca559a0643f83f1f99c90fa28691ae069983f57f521927df688" in 11.264s (11.264s including waiting). Image size: 299671472 bytes.' metadata: creationTimestamp: "2026-04-16T18:15:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:15:42Z" name: isvc-sklearn-graph-2-predictor-d76b874f9-gp6ts.18a6e90c82d714ee namespace: kserve-ci-e2e-test resourceVersion: "14822" uid: 5a54ba38-7625-4384-9bf7-792dff374891 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:15:44Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-d76b874f9-gp6ts namespace: kserve-ci-e2e-test resourceVersion: "14686" uid: 6e7fd811-a17a-4ab8-8c18-e26dda756487 kind: Event lastTimestamp: "2026-04-16T18:15:44Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T18:15:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:15:44Z" name: isvc-sklearn-graph-2-predictor-d76b874f9-gp6ts.18a6e90ce50b0299 namespace: kserve-ci-e2e-test resourceVersion: "14839" uid: 3ac93577-eef5-45ab-8add-b67584de971b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:15:44Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-d76b874f9-gp6ts namespace: kserve-ci-e2e-test resourceVersion: "14686" uid: 6e7fd811-a17a-4ab8-8c18-e26dda756487 kind: Event lastTimestamp: "2026-04-16T18:15:44Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T18:15:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:15:44Z" name: isvc-sklearn-graph-2-predictor-d76b874f9-gp6ts.18a6e90ce58774d7 namespace: kserve-ci-e2e-test resourceVersion: "14840" uid: 1a1d74f8-74d9-47b0-b059-13c1e3656e92 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:15:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-d76b874f9-gp6ts namespace: kserve-ci-e2e-test resourceVersion: "14686" uid: 6e7fd811-a17a-4ab8-8c18-e26dda756487 kind: Event lastTimestamp: "2026-04-16T18:15:49Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1412" metadata: creationTimestamp: "2026-04-16T18:15:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:15:49Z" name: isvc-sklearn-graph-2-predictor-d76b874f9-gp6ts.18a6e90dfa629828 namespace: kserve-ci-e2e-test resourceVersion: "14909" uid: 4b7df6ea-1ab6-4ff4-9ddb-25b010b8e097 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:15:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-d76b874f9-gp6ts namespace: kserve-ci-e2e-test resourceVersion: "14686" uid: 6e7fd811-a17a-4ab8-8c18-e26dda756487 kind: Event lastTimestamp: "2026-04-16T18:15:56Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1412" in 6.99s (6.99s including waiting). Image size: 1560610740 bytes.' metadata: creationTimestamp: "2026-04-16T18:15:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:15:56Z" name: isvc-sklearn-graph-2-predictor-d76b874f9-gp6ts.18a6e90f9b13c1ea namespace: kserve-ci-e2e-test resourceVersion: "14970" uid: ef76eae4-75a6-4f05-b598-15702680a0aa reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:15:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-d76b874f9-gp6ts namespace: kserve-ci-e2e-test resourceVersion: "14686" uid: 6e7fd811-a17a-4ab8-8c18-e26dda756487 kind: Event lastTimestamp: "2026-04-16T18:15:56Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:15:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:15:56Z" name: isvc-sklearn-graph-2-predictor-d76b874f9-gp6ts.18a6e90fa0172ec2 namespace: kserve-ci-e2e-test resourceVersion: "14971" uid: ca97cfca-6871-4c94-9a8c-01c96cfbaf80 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:15:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-d76b874f9-gp6ts namespace: kserve-ci-e2e-test resourceVersion: "14686" uid: 6e7fd811-a17a-4ab8-8c18-e26dda756487 kind: Event lastTimestamp: "2026-04-16T18:15:56Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:15:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:15:56Z" name: isvc-sklearn-graph-2-predictor-d76b874f9-gp6ts.18a6e90fa0a2c9f8 namespace: kserve-ci-e2e-test resourceVersion: "14972" uid: da571d3f-8dab-4b4b-a411-022ca11859e1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-16T18:15:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-d76b874f9-gp6ts namespace: kserve-ci-e2e-test resourceVersion: "14686" uid: 6e7fd811-a17a-4ab8-8c18-e26dda756487 kind: Event lastTimestamp: "2026-04-16T18:17:33Z" message: 'Readiness probe failed: dial tcp 10.132.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:15:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:17:33Z" name: isvc-sklearn-graph-2-predictor-d76b874f9-gp6ts.18a6e90fd991ae95 namespace: kserve-ci-e2e-test resourceVersion: "16170" uid: 9eee2942-2a89-4967-8a8b-49730c3e585a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:17:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-d76b874f9-gp6ts namespace: kserve-ci-e2e-test resourceVersion: "14686" uid: 6e7fd811-a17a-4ab8-8c18-e26dda756487 kind: Event lastTimestamp: "2026-04-16T18:17:30Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:17:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:17:30Z" name: isvc-sklearn-graph-2-predictor-d76b874f9-gp6ts.18a6e9259f14d6bc namespace: kserve-ci-e2e-test resourceVersion: "16120" uid: 8872d1f2-0967-46ea-af94-fdcd995180e6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:15:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-d76b874f9 namespace: kserve-ci-e2e-test resourceVersion: "14669" uid: ed0fe928-0ed8-4d70-8fc7-f24af360c800 kind: Event lastTimestamp: "2026-04-16T18:15:31Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-d76b874f9-gp6ts' metadata: creationTimestamp: "2026-04-16T18:15:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:15:31Z" name: isvc-sklearn-graph-2-predictor-d76b874f9.18a6e909c6c7bfdb namespace: kserve-ci-e2e-test resourceVersion: "14687" uid: 67a24704-0ad1-4f71-b5d8-042d9333eb2a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:15:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14662" uid: 846cd5a6-ab35-4553-b6ad-f0977fa27662 kind: Event lastTimestamp: "2026-04-16T18:15:31Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-d76b874f9 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:15:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:15:31Z" name: isvc-sklearn-graph-2-predictor.18a6e909c0b86c5e namespace: kserve-ci-e2e-test resourceVersion: "14674" uid: 9eb1c809-0fcf-4717-8568-1eb7a3271897 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T18:15:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14666" uid: 6126cb11-f160-48d0-97f3-40a7a93ff1f0 kind: Event lastTimestamp: "2026-04-16T18:16:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:15:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:16:31Z" name: isvc-sklearn-graph-2-predictor.18a6e90d3f896937 namespace: kserve-ci-e2e-test resourceVersion: "15267" uid: afa23de8-fa27-41a4-b039-364ccac6f397 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T18:15:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14666" uid: 6126cb11-f160-48d0-97f3-40a7a93ff1f0 kind: Event lastTimestamp: "2026-04-16T18:16:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:15:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:16:31Z" name: isvc-sklearn-graph-2-predictor.18a6e90d4007839a namespace: kserve-ci-e2e-test resourceVersion: "15268" uid: a8637b91-0db9-45cb-85fd-8501ec8ac8b0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:16:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14885" uid: 6126cb11-f160-48d0-97f3-40a7a93ff1f0 kind: Event lastTimestamp: "2026-04-16T18:17:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:16:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:17:16Z" name: isvc-sklearn-graph-2-predictor.18a6e91b3b47ba26 namespace: kserve-ci-e2e-test resourceVersion: "15839" uid: ee8bc3df-ad14-4e9a-99b3-17d9e52b8b7d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:16:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14885" uid: 6126cb11-f160-48d0-97f3-40a7a93ff1f0 kind: Event lastTimestamp: "2026-04-16T18:17:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:16:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:17:16Z" name: isvc-sklearn-graph-2-predictor.18a6e91b3bbe5d03 namespace: kserve-ci-e2e-test resourceVersion: "15840" uid: 38289321-fc79-496e-8d88-bef48420f84f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-16T18:15:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14651" uid: cffde902-237a-4a8a-90fa-085e147d03e6 kind: Event lastTimestamp: "2026-04-16T18:17:13Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:15:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:17:13Z" name: isvc-sklearn-graph-2.18a6e909c0c4a41e namespace: kserve-ci-e2e-test resourceVersion: "15804" uid: b2959fd0-bfb5-47b5-8aab-b612734ade23 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:17:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "15802" uid: cffde902-237a-4a8a-90fa-085e147d03e6 kind: Event lastTimestamp: "2026-04-16T18:17:13Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-04-16T18:17:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:17:13Z" name: isvc-sklearn-graph-2.18a6e921993d7495 namespace: kserve-ci-e2e-test resourceVersion: "15803" uid: 4d62ee3c-8650-415a-b902-9d62547fbcf6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:15:31.143975Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-ll9ps namespace: kserve-ci-e2e-test resourceVersion: "14661" uid: 2847bd53-2a20-4981-a0d9-29a0392aef9c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-784cb989b8-ll9ps to ip-10-0-134-133.ec2.internal metadata: creationTimestamp: "2026-04-16T18:15:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:15:31Z" name: isvc-xgboost-graph-predictor-784cb989b8-ll9ps.18a6e909c0ddd9cd namespace: kserve-ci-e2e-test resourceVersion: "14672" uid: 6f6b914b-8eca-420a-8562-3de56a761cd2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4b7bbd69-nffhf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:15:31Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-ll9ps namespace: kserve-ci-e2e-test resourceVersion: "14680" uid: 2847bd53-2a20-4981-a0d9-29a0392aef9c kind: Event lastTimestamp: "2026-04-16T18:15:31Z" message: Add eth0 [10.132.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:15:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:15:31Z" name: isvc-xgboost-graph-predictor-784cb989b8-ll9ps.18a6e909daa2d193 namespace: kserve-ci-e2e-test resourceVersion: "14706" uid: 70e63390-22b7-4054-a091-0e703cbdb1eb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:15:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-ll9ps namespace: kserve-ci-e2e-test resourceVersion: "14668" uid: 2847bd53-2a20-4981-a0d9-29a0392aef9c kind: Event lastTimestamp: "2026-04-16T18:15:31Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:b1408675ea4fdca559a0643f83f1f99c90fa28691ae069983f57f521927df688" metadata: creationTimestamp: "2026-04-16T18:15:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:15:31Z" name: isvc-xgboost-graph-predictor-784cb989b8-ll9ps.18a6e909dd09f857 namespace: kserve-ci-e2e-test resourceVersion: "14709" uid: 3188f1c0-e054-4fdb-92cd-fb5c7c09e35f reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:15:44Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-ll9ps namespace: kserve-ci-e2e-test resourceVersion: "14668" uid: 2847bd53-2a20-4981-a0d9-29a0392aef9c kind: Event lastTimestamp: "2026-04-16T18:15:44Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:b1408675ea4fdca559a0643f83f1f99c90fa28691ae069983f57f521927df688" in 12.941s (12.941s including waiting). Image size: 299671472 bytes.' metadata: creationTimestamp: "2026-04-16T18:15:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:15:44Z" name: isvc-xgboost-graph-predictor-784cb989b8-ll9ps.18a6e90ce0631ec6 namespace: kserve-ci-e2e-test resourceVersion: "14836" uid: cb1df642-a896-417a-93b2-73868c70cd25 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:15:44Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-ll9ps namespace: kserve-ci-e2e-test resourceVersion: "14668" uid: 2847bd53-2a20-4981-a0d9-29a0392aef9c kind: Event lastTimestamp: "2026-04-16T18:15:44Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-16T18:15:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:15:44Z" name: isvc-xgboost-graph-predictor-784cb989b8-ll9ps.18a6e90ce4ffcc53 namespace: kserve-ci-e2e-test resourceVersion: "14838" uid: 95b7e7eb-be90-4888-a114-56e05b4df9ae reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:15:44Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-ll9ps namespace: kserve-ci-e2e-test resourceVersion: "14668" uid: 2847bd53-2a20-4981-a0d9-29a0392aef9c kind: Event lastTimestamp: "2026-04-16T18:15:44Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-16T18:15:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:15:44Z" name: isvc-xgboost-graph-predictor-784cb989b8-ll9ps.18a6e90ce5931c35 namespace: kserve-ci-e2e-test resourceVersion: "14841" uid: 37fd4f01-46a5-4657-af15-014b3e71665e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:15:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-ll9ps namespace: kserve-ci-e2e-test resourceVersion: "14668" uid: 2847bd53-2a20-4981-a0d9-29a0392aef9c kind: Event lastTimestamp: "2026-04-16T18:15:49Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-04-16T18:15:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:15:49Z" name: isvc-xgboost-graph-predictor-784cb989b8-ll9ps.18a6e90dfa7905a6 namespace: kserve-ci-e2e-test resourceVersion: "14910" uid: 10a01ace-7000-4883-8591-2007fd65adfe reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:16:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-ll9ps namespace: kserve-ci-e2e-test resourceVersion: "14668" uid: 2847bd53-2a20-4981-a0d9-29a0392aef9c kind: Event lastTimestamp: "2026-04-16T18:16:07Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 18.403s (18.403s including waiting). Image size: 1306229499 bytes.' metadata: creationTimestamp: "2026-04-16T18:16:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:16:07Z" name: isvc-xgboost-graph-predictor-784cb989b8-ll9ps.18a6e912436cfe6b namespace: kserve-ci-e2e-test resourceVersion: "15069" uid: 8f52c328-e16f-472b-8836-9d8a25eba36e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:16:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-ll9ps namespace: kserve-ci-e2e-test resourceVersion: "14668" uid: 2847bd53-2a20-4981-a0d9-29a0392aef9c kind: Event lastTimestamp: "2026-04-16T18:16:07Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:16:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:16:07Z" name: isvc-xgboost-graph-predictor-784cb989b8-ll9ps.18a6e9124acc7938 namespace: kserve-ci-e2e-test resourceVersion: "15071" uid: 9cfd0407-458c-4e4d-85cb-ff9d97409042 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:16:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-ll9ps namespace: kserve-ci-e2e-test resourceVersion: "14668" uid: 2847bd53-2a20-4981-a0d9-29a0392aef9c kind: Event lastTimestamp: "2026-04-16T18:16:07Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:16:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:16:07Z" name: isvc-xgboost-graph-predictor-784cb989b8-ll9ps.18a6e9124be3c8ad namespace: kserve-ci-e2e-test resourceVersion: "15072" uid: 23453117-4f97-41ef-a0f0-23f408d16d46 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T18:16:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-ll9ps namespace: kserve-ci-e2e-test resourceVersion: "14668" uid: 2847bd53-2a20-4981-a0d9-29a0392aef9c kind: Event lastTimestamp: "2026-04-16T18:16:59Z" message: 'Readiness probe failed: dial tcp 10.132.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:16:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:16:59Z" name: isvc-xgboost-graph-predictor-784cb989b8-ll9ps.18a6e9126bb8cb35 namespace: kserve-ci-e2e-test resourceVersion: "15543" uid: ea80e716-8a9f-418c-b718-28e7a853589d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:17:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-784cb989b8-ll9ps namespace: kserve-ci-e2e-test resourceVersion: "14668" uid: 2847bd53-2a20-4981-a0d9-29a0392aef9c kind: Event lastTimestamp: "2026-04-16T18:17:30Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:17:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:17:30Z" name: isvc-xgboost-graph-predictor-784cb989b8-ll9ps.18a6e925988667f1 namespace: kserve-ci-e2e-test resourceVersion: "16096" uid: 4fbf05b7-659b-4c09-83a7-8f9d8a5022be reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-133.ec2.internal source: component: kubelet host: ip-10-0-134-133.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:15:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-784cb989b8 namespace: kserve-ci-e2e-test resourceVersion: "14645" uid: e410cc78-a131-4500-ac03-4dca6c6e24e5 kind: Event lastTimestamp: "2026-04-16T18:15:31Z" message: 'Created pod: isvc-xgboost-graph-predictor-784cb989b8-ll9ps' metadata: creationTimestamp: "2026-04-16T18:15:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:15:31Z" name: isvc-xgboost-graph-predictor-784cb989b8.18a6e909bfdaf421 namespace: kserve-ci-e2e-test resourceVersion: "14667" uid: 849dae9c-ce84-45b6-9b12-6d526fe8f1c8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:15:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14639" uid: f1cc7a72-67e6-4511-9eff-e3dd9326853c kind: Event lastTimestamp: "2026-04-16T18:15:30Z" message: Scaled up replica set isvc-xgboost-graph-predictor-784cb989b8 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:15:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:15:30Z" name: isvc-xgboost-graph-predictor.18a6e909b29ef977 namespace: kserve-ci-e2e-test resourceVersion: "14647" uid: 7c9c53bf-b36e-4bc2-9d0e-6303fd46beec reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T18:15:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14642" uid: 5904f650-118b-46b4-9e21-537e28443e79 kind: Event lastTimestamp: "2026-04-16T18:16:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:15:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:16:30Z" name: isvc-xgboost-graph-predictor.18a6e90d2fc3a416 namespace: kserve-ci-e2e-test resourceVersion: "15264" uid: f5f8bea4-3648-465a-be7a-848adcdb7d35 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T18:15:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14642" uid: 5904f650-118b-46b4-9e21-537e28443e79 kind: Event lastTimestamp: "2026-04-16T18:16:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:15:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:16:30Z" name: isvc-xgboost-graph-predictor.18a6e90d302faac9 namespace: kserve-ci-e2e-test resourceVersion: "15265" uid: 1df4253e-dbc4-4ef0-a2c2-e4ca65de0469 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:16:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14880" uid: 5904f650-118b-46b4-9e21-537e28443e79 kind: Event lastTimestamp: "2026-04-16T18:17:15Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:16:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:17:15Z" name: isvc-xgboost-graph-predictor.18a6e91b2b5b3376 namespace: kserve-ci-e2e-test resourceVersion: "15835" uid: 9e093d04-7b7d-434c-bacc-c46f4fe5cfa9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:16:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14880" uid: 5904f650-118b-46b4-9e21-537e28443e79 kind: Event lastTimestamp: "2026-04-16T18:17:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:16:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:17:15Z" name: isvc-xgboost-graph-predictor.18a6e91b2bc67a3b namespace: kserve-ci-e2e-test resourceVersion: "15836" uid: 2ce8181f-d0ab-4ff1-bd3d-edcc2c29ec35 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-16T18:15:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14623" uid: 8d50cea1-e9cb-4e34-bfb0-c7372a734fbc kind: Event lastTimestamp: "2026-04-16T18:17:09Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:15:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:17:09Z" name: isvc-xgboost-graph.18a6e909b10ce1c5 namespace: kserve-ci-e2e-test resourceVersion: "15767" uid: 442632d8-8b81-4649-9f90-a5d4cd32a566 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:17:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15765" uid: 8d50cea1-e9cb-4e34-bfb0-c7372a734fbc kind: Event lastTimestamp: "2026-04-16T18:17:09Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-04-16T18:17:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:17:09Z" name: isvc-xgboost-graph.18a6e920a4c0d2b3 namespace: kserve-ci-e2e-test resourceVersion: "15766" uid: 8b23057a-db25-45c8-add0-e96e9fe3f7d3 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:17:20.292919Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-77968cf4b8-qdr99 namespace: kserve-ci-e2e-test resourceVersion: "15879" uid: 530f1448-da66-4015-8ce1-e8ea6e692cea kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-77968cf4b8-qdr99 to ip-10-0-128-59.ec2.internal metadata: creationTimestamp: "2026-04-16T18:17:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:17:20Z" name: model-chainer-77968cf4b8-qdr99.18a6e9232aa696bd namespace: kserve-ci-e2e-test resourceVersion: "15884" uid: 697096c8-926b-4eac-bab9-1048dcdf50f2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4b7bbd69-nffhf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:17:20Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-77968cf4b8-qdr99 namespace: kserve-ci-e2e-test resourceVersion: "15887" uid: 530f1448-da66-4015-8ce1-e8ea6e692cea kind: Event lastTimestamp: "2026-04-16T18:17:20Z" message: Add eth0 [10.134.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:17:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:17:20Z" name: model-chainer-77968cf4b8-qdr99.18a6e92343772eb9 namespace: kserve-ci-e2e-test resourceVersion: "15898" uid: d747f16d-dd55-4857-aff5-6b798b7d1123 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:17:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-77968cf4b8-qdr99 namespace: kserve-ci-e2e-test resourceVersion: "15882" uid: 530f1448-da66-4015-8ce1-e8ea6e692cea kind: Event lastTimestamp: "2026-04-16T18:17:20Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:6060da2148ba78c06d6de96222351f1e5f7d3ada4027ac02142aa0e1928e4f29" already present on machine metadata: creationTimestamp: "2026-04-16T18:17:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:17:20Z" name: model-chainer-77968cf4b8-qdr99.18a6e92344ccd666 namespace: kserve-ci-e2e-test resourceVersion: "15901" uid: 3c9dfe0a-91c8-49cc-b6ae-cf7d4245db99 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:17:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-77968cf4b8-qdr99 namespace: kserve-ci-e2e-test resourceVersion: "15882" uid: 530f1448-da66-4015-8ce1-e8ea6e692cea kind: Event lastTimestamp: "2026-04-16T18:17:20Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-04-16T18:17:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:17:20Z" name: model-chainer-77968cf4b8-qdr99.18a6e92347a08d37 namespace: kserve-ci-e2e-test resourceVersion: "15902" uid: e1f7aa9c-e576-4586-9619-e0bd1f105ab2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:17:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-77968cf4b8-qdr99 namespace: kserve-ci-e2e-test resourceVersion: "15882" uid: 530f1448-da66-4015-8ce1-e8ea6e692cea kind: Event lastTimestamp: "2026-04-16T18:17:20Z" message: Started container model-chainer metadata: creationTimestamp: "2026-04-16T18:17:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:17:20Z" name: model-chainer-77968cf4b8-qdr99.18a6e92347ff22ec namespace: kserve-ci-e2e-test resourceVersion: "15903" uid: 1f721345-3f2f-4ab2-b318-05b3c2efd9cf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:17:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-77968cf4b8-qdr99 namespace: kserve-ci-e2e-test resourceVersion: "15882" uid: 530f1448-da66-4015-8ce1-e8ea6e692cea kind: Event lastTimestamp: "2026-04-16T18:17:30Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-04-16T18:17:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:17:30Z" name: model-chainer-77968cf4b8-qdr99.18a6e925848b82d8 namespace: kserve-ci-e2e-test resourceVersion: "16012" uid: 3145167c-266d-4fb4-96ca-97a8c98f00f1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:17:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-77968cf4b8-qdr99 namespace: kserve-ci-e2e-test resourceVersion: "15882" uid: 530f1448-da66-4015-8ce1-e8ea6e692cea kind: Event lastTimestamp: "2026-04-16T18:17:57Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T18:17:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:17:57Z" name: model-chainer-77968cf4b8-qdr99.18a6e925f06cac36 namespace: kserve-ci-e2e-test resourceVersion: "16381" uid: f6174676-2bd2-476b-b17e-fd321433cd18 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:17:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-77968cf4b8 namespace: kserve-ci-e2e-test resourceVersion: "15876" uid: 32082a99-4f13-4cdb-bfb0-0ac6c0632716 kind: Event lastTimestamp: "2026-04-16T18:17:20Z" message: 'Created pod: model-chainer-77968cf4b8-qdr99' metadata: creationTimestamp: "2026-04-16T18:17:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:17:20Z" name: model-chainer-77968cf4b8.18a6e9232a18c88e namespace: kserve-ci-e2e-test resourceVersion: "15880" uid: 974c415d-a6dd-48b3-b88a-075c82975d32 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:17:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15874" uid: 04b7a1a1-90f3-4a82-b82a-ede81333d064 kind: Event lastTimestamp: "2026-04-16T18:17:20Z" message: Scaled up replica set model-chainer-77968cf4b8 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:17:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:17:20Z" name: model-chainer.18a6e9232944933a namespace: kserve-ci-e2e-test resourceVersion: "15877" uid: 1f7225e7-0fbe-4ba4-82e1-7b0bd50393bf reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:17:20Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15890" uid: a0210fb2-e159-4281-89da-563a1b20d5b4 kind: Event lastTimestamp: "2026-04-16T18:17:20Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-04-16T18:17:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:17:20Z" name: model-chainer.18a6e9232f655a19 namespace: kserve-ci-e2e-test resourceVersion: "15892" uid: c4a094b8-3409-4478-8740-9be281315995 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:18:40.598761Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-0403e-6dd9d7fbd6-55qr5 namespace: kserve-ci-e2e-test resourceVersion: "16780" uid: ce6c77c0-51fa-4864-87f4-072aae4ca35e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-0403e-6dd9d7fbd6-55qr5 to ip-10-0-128-59.ec2.internal metadata: creationTimestamp: "2026-04-16T18:18:40Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:18:40Z" name: sequence-graph-0403e-6dd9d7fbd6-55qr5.18a6e935dd4094b6 namespace: kserve-ci-e2e-test resourceVersion: "16785" uid: f433c19c-c64f-418e-aa08-582f76b40d33 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4b7bbd69-nffhf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:41Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-0403e-6dd9d7fbd6-55qr5 namespace: kserve-ci-e2e-test resourceVersion: "16788" uid: ce6c77c0-51fa-4864-87f4-072aae4ca35e kind: Event lastTimestamp: "2026-04-16T18:18:41Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:18:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:18:41Z" name: sequence-graph-0403e-6dd9d7fbd6-55qr5.18a6e935f633047d namespace: kserve-ci-e2e-test resourceVersion: "16803" uid: f6a6afec-9fe2-46ef-b7ee-1572b398e26d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-0403e} kind: Pod name: sequence-graph-0403e-6dd9d7fbd6-55qr5 namespace: kserve-ci-e2e-test resourceVersion: "16782" uid: ce6c77c0-51fa-4864-87f4-072aae4ca35e kind: Event lastTimestamp: "2026-04-16T18:18:41Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:6060da2148ba78c06d6de96222351f1e5f7d3ada4027ac02142aa0e1928e4f29" already present on machine metadata: creationTimestamp: "2026-04-16T18:18:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:18:41Z" name: sequence-graph-0403e-6dd9d7fbd6-55qr5.18a6e935f766869c namespace: kserve-ci-e2e-test resourceVersion: "16805" uid: 11831a3f-ae4f-4dd7-a18f-43a6438cc0a4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-0403e} kind: Pod name: sequence-graph-0403e-6dd9d7fbd6-55qr5 namespace: kserve-ci-e2e-test resourceVersion: "16782" uid: ce6c77c0-51fa-4864-87f4-072aae4ca35e kind: Event lastTimestamp: "2026-04-16T18:18:41Z" message: 'Created container: sequence-graph-0403e' metadata: creationTimestamp: "2026-04-16T18:18:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:18:41Z" name: sequence-graph-0403e-6dd9d7fbd6-55qr5.18a6e935fa3529e2 namespace: kserve-ci-e2e-test resourceVersion: "16807" uid: d74dfcda-50de-4cd7-bf13-023137ca2f66 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-0403e} kind: Pod name: sequence-graph-0403e-6dd9d7fbd6-55qr5 namespace: kserve-ci-e2e-test resourceVersion: "16782" uid: ce6c77c0-51fa-4864-87f4-072aae4ca35e kind: Event lastTimestamp: "2026-04-16T18:18:41Z" message: Started container sequence-graph-0403e metadata: creationTimestamp: "2026-04-16T18:18:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:18:41Z" name: sequence-graph-0403e-6dd9d7fbd6-55qr5.18a6e935faa82b33 namespace: kserve-ci-e2e-test resourceVersion: "16808" uid: 5626e830-fe04-4153-b751-9fccebd8f232 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-0403e} kind: Pod name: sequence-graph-0403e-6dd9d7fbd6-55qr5 namespace: kserve-ci-e2e-test resourceVersion: "16782" uid: ce6c77c0-51fa-4864-87f4-072aae4ca35e kind: Event lastTimestamp: "2026-04-16T18:26:55Z" message: Stopping container sequence-graph-0403e metadata: creationTimestamp: "2026-04-16T18:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:26:55Z" name: sequence-graph-0403e-6dd9d7fbd6-55qr5.18a6e9a9134be617 namespace: kserve-ci-e2e-test resourceVersion: "20343" uid: 67ce8048-0b30-4496-aded-990076f9748b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:26:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-0403e} kind: Pod name: sequence-graph-0403e-6dd9d7fbd6-55qr5 namespace: kserve-ci-e2e-test resourceVersion: "16782" uid: ce6c77c0-51fa-4864-87f4-072aae4ca35e kind: Event lastTimestamp: "2026-04-16T18:27:22Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T18:26:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:27:22Z" name: sequence-graph-0403e-6dd9d7fbd6-55qr5.18a6e9a98bad0dd4 namespace: kserve-ci-e2e-test resourceVersion: "20690" uid: 02c6f61b-7ec6-482b-943e-4554bfecf614 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:40Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-0403e-6dd9d7fbd6 namespace: kserve-ci-e2e-test resourceVersion: "16777" uid: ac02426f-d4c4-49bf-92fd-ed735231c933 kind: Event lastTimestamp: "2026-04-16T18:18:40Z" message: 'Created pod: sequence-graph-0403e-6dd9d7fbd6-55qr5' metadata: creationTimestamp: "2026-04-16T18:18:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:18:40Z" name: sequence-graph-0403e-6dd9d7fbd6.18a6e935dcd0c6ae namespace: kserve-ci-e2e-test resourceVersion: "16781" uid: 781d845f-99e8-4e12-83d4-e23a52df2215 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:40Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-0403e namespace: kserve-ci-e2e-test resourceVersion: "16775" uid: 57669af2-5dda-4544-87a0-8976ebcc0ef7 kind: Event lastTimestamp: "2026-04-16T18:18:40Z" message: Scaled up replica set sequence-graph-0403e-6dd9d7fbd6 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:18:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:18:40Z" name: sequence-graph-0403e.18a6e935dc072abc namespace: kserve-ci-e2e-test resourceVersion: "16778" uid: 8f074127-5092-4693-ba6a-69f472ccd77c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:40Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-0403e namespace: kserve-ci-e2e-test resourceVersion: "16793" uid: 1a493d2a-abc3-441b-adc6-4c90937649f0 kind: Event lastTimestamp: "2026-04-16T18:18:40Z" message: InferenceGraph [sequence-graph-0403e] is Ready metadata: creationTimestamp: "2026-04-16T18:18:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:18:40Z" name: sequence-graph-0403e.18a6e935e2ba7981 namespace: kserve-ci-e2e-test resourceVersion: "16795" uid: 74730097-9efd-4663-a40c-fd9dfb46b819 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:18:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-0403e namespace: kserve-ci-e2e-test resourceVersion: "16776" uid: 419aaf37-502f-4be1-b519-bb3f926ea7d3 kind: Event lastTimestamp: "2026-04-16T18:19:25Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:18:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:19:25Z" name: sequence-graph-0403e.18a6e9395a912fed namespace: kserve-ci-e2e-test resourceVersion: "17125" uid: dc1718cd-884e-4516-806a-09273810f1c7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:18:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-0403e namespace: kserve-ci-e2e-test resourceVersion: "16776" uid: 419aaf37-502f-4be1-b519-bb3f926ea7d3 kind: Event lastTimestamp: "2026-04-16T18:19:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:18:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:19:25Z" name: sequence-graph-0403e.18a6e9395aea08a5 namespace: kserve-ci-e2e-test resourceVersion: "17126" uid: e5b8390c-68e4-47cb-827b-78d176580f5f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:28:05.589245Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-516af-7cdf98dc78-cwt5d namespace: kserve-ci-e2e-test resourceVersion: "21234" uid: c7e9eee7-daf5-4fa1-86a1-469d3d0ae7af kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-516af-7cdf98dc78-cwt5d to ip-10-0-128-59.ec2.internal metadata: creationTimestamp: "2026-04-16T18:28:05Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:28:05Z" name: sequence-graph-516af-7cdf98dc78-cwt5d.18a6e9b9694f2122 namespace: kserve-ci-e2e-test resourceVersion: "21239" uid: 76a0d6e8-aa63-4951-803d-b3a2b274f12a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4b7bbd69-nffhf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:06Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-516af-7cdf98dc78-cwt5d namespace: kserve-ci-e2e-test resourceVersion: "21244" uid: c7e9eee7-daf5-4fa1-86a1-469d3d0ae7af kind: Event lastTimestamp: "2026-04-16T18:28:06Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:28:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:28:06Z" name: sequence-graph-516af-7cdf98dc78-cwt5d.18a6e9b9825966f7 namespace: kserve-ci-e2e-test resourceVersion: "21255" uid: a564fde5-d4f6-4223-b0d0-f72e7a3e535d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-516af} kind: Pod name: sequence-graph-516af-7cdf98dc78-cwt5d namespace: kserve-ci-e2e-test resourceVersion: "21236" uid: c7e9eee7-daf5-4fa1-86a1-469d3d0ae7af kind: Event lastTimestamp: "2026-04-16T18:28:06Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:6060da2148ba78c06d6de96222351f1e5f7d3ada4027ac02142aa0e1928e4f29" already present on machine metadata: creationTimestamp: "2026-04-16T18:28:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:28:06Z" name: sequence-graph-516af-7cdf98dc78-cwt5d.18a6e9b983af9fb3 namespace: kserve-ci-e2e-test resourceVersion: "21257" uid: f26abd36-afb9-473e-983a-c186addc09f0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-516af} kind: Pod name: sequence-graph-516af-7cdf98dc78-cwt5d namespace: kserve-ci-e2e-test resourceVersion: "21236" uid: c7e9eee7-daf5-4fa1-86a1-469d3d0ae7af kind: Event lastTimestamp: "2026-04-16T18:28:06Z" message: 'Created container: sequence-graph-516af' metadata: creationTimestamp: "2026-04-16T18:28:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:28:06Z" name: sequence-graph-516af-7cdf98dc78-cwt5d.18a6e9b98689dba1 namespace: kserve-ci-e2e-test resourceVersion: "21258" uid: af9aa0ad-2479-4730-9bff-d86fa1df977d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-516af} kind: Pod name: sequence-graph-516af-7cdf98dc78-cwt5d namespace: kserve-ci-e2e-test resourceVersion: "21236" uid: c7e9eee7-daf5-4fa1-86a1-469d3d0ae7af kind: Event lastTimestamp: "2026-04-16T18:28:06Z" message: Started container sequence-graph-516af metadata: creationTimestamp: "2026-04-16T18:28:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:28:06Z" name: sequence-graph-516af-7cdf98dc78-cwt5d.18a6e9b986f957d5 namespace: kserve-ci-e2e-test resourceVersion: "21259" uid: 85094c95-f48f-463b-9527-68fe2fa53d62 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-516af} kind: Pod name: sequence-graph-516af-7cdf98dc78-cwt5d namespace: kserve-ci-e2e-test resourceVersion: "21236" uid: c7e9eee7-daf5-4fa1-86a1-469d3d0ae7af kind: Event lastTimestamp: "2026-04-16T18:28:15Z" message: Stopping container sequence-graph-516af metadata: creationTimestamp: "2026-04-16T18:28:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:28:15Z" name: sequence-graph-516af-7cdf98dc78-cwt5d.18a6e9bbc28c1d2b namespace: kserve-ci-e2e-test resourceVersion: "21359" uid: 193ac59f-e820-4b65-be75-9436ea015b3d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:28:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-516af} kind: Pod name: sequence-graph-516af-7cdf98dc78-cwt5d namespace: kserve-ci-e2e-test resourceVersion: "21236" uid: c7e9eee7-daf5-4fa1-86a1-469d3d0ae7af kind: Event lastTimestamp: "2026-04-16T18:28:43Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T18:28:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:28:43Z" name: sequence-graph-516af-7cdf98dc78-cwt5d.18a6e9bc51095795 namespace: kserve-ci-e2e-test resourceVersion: "21711" uid: a308d9e3-3194-4a6e-b851-da572c02dd15 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:05Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-516af-7cdf98dc78 namespace: kserve-ci-e2e-test resourceVersion: "21231" uid: b714b0d9-95cc-4040-a804-f9dd92f964eb kind: Event lastTimestamp: "2026-04-16T18:28:05Z" message: 'Created pod: sequence-graph-516af-7cdf98dc78-cwt5d' metadata: creationTimestamp: "2026-04-16T18:28:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:28:05Z" name: sequence-graph-516af-7cdf98dc78.18a6e9b968c2795f namespace: kserve-ci-e2e-test resourceVersion: "21235" uid: e606fba2-9367-4ba0-8879-d1a3c1c072cb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:05Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-516af namespace: kserve-ci-e2e-test resourceVersion: "21229" uid: 4a09008a-fba8-45b7-b5c2-dbce2d464e1b kind: Event lastTimestamp: "2026-04-16T18:28:05Z" message: Scaled up replica set sequence-graph-516af-7cdf98dc78 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:28:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:28:05Z" name: sequence-graph-516af.18a6e9b96822622a namespace: kserve-ci-e2e-test resourceVersion: "21232" uid: 9ef58cf7-c571-4283-9fb3-c883db116213 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:05Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-516af namespace: kserve-ci-e2e-test resourceVersion: "21246" uid: 8fca8320-6520-449f-88b7-3104e7a484fb kind: Event lastTimestamp: "2026-04-16T18:28:05Z" message: InferenceGraph [sequence-graph-516af] is Ready metadata: creationTimestamp: "2026-04-16T18:28:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:28:05Z" name: sequence-graph-516af.18a6e9b96cb5cd75 namespace: kserve-ci-e2e-test resourceVersion: "21248" uid: 2fdf214f-4729-41be-8324-9718998f144d reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:29:25.919529Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-ed24b-7c6b74f4df-gtjnk namespace: kserve-ci-e2e-test resourceVersion: "22105" uid: 45d446be-c5bb-4512-b301-7036b270391e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-ed24b-7c6b74f4df-gtjnk to ip-10-0-128-59.ec2.internal metadata: creationTimestamp: "2026-04-16T18:29:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:29:25Z" name: sequence-graph-ed24b-7c6b74f4df-gtjnk.18a6e9cc1d5dff96 namespace: kserve-ci-e2e-test resourceVersion: "22110" uid: 15c01cb0-3cab-4984-9dc7-1f23c36988ee reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4b7bbd69-nffhf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:29:26Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-ed24b-7c6b74f4df-gtjnk namespace: kserve-ci-e2e-test resourceVersion: "22113" uid: 45d446be-c5bb-4512-b301-7036b270391e kind: Event lastTimestamp: "2026-04-16T18:29:26Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:29:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:29:26Z" name: sequence-graph-ed24b-7c6b74f4df-gtjnk.18a6e9cc363462d7 namespace: kserve-ci-e2e-test resourceVersion: "22127" uid: 1fee7f96-00a0-4a4f-8fa4-935a93d5559a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:29:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-ed24b} kind: Pod name: sequence-graph-ed24b-7c6b74f4df-gtjnk namespace: kserve-ci-e2e-test resourceVersion: "22108" uid: 45d446be-c5bb-4512-b301-7036b270391e kind: Event lastTimestamp: "2026-04-16T18:29:26Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:6060da2148ba78c06d6de96222351f1e5f7d3ada4027ac02142aa0e1928e4f29" already present on machine metadata: creationTimestamp: "2026-04-16T18:29:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:29:26Z" name: sequence-graph-ed24b-7c6b74f4df-gtjnk.18a6e9cc376f24ab namespace: kserve-ci-e2e-test resourceVersion: "22129" uid: 7b20dac1-028d-41b8-97cf-92617232ef5e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:29:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-ed24b} kind: Pod name: sequence-graph-ed24b-7c6b74f4df-gtjnk namespace: kserve-ci-e2e-test resourceVersion: "22108" uid: 45d446be-c5bb-4512-b301-7036b270391e kind: Event lastTimestamp: "2026-04-16T18:29:26Z" message: 'Created container: sequence-graph-ed24b' metadata: creationTimestamp: "2026-04-16T18:29:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:29:26Z" name: sequence-graph-ed24b-7c6b74f4df-gtjnk.18a6e9cc3a723fa2 namespace: kserve-ci-e2e-test resourceVersion: "22130" uid: f6480c74-ff4a-4ec8-b33a-4c32beb27713 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:29:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-ed24b} kind: Pod name: sequence-graph-ed24b-7c6b74f4df-gtjnk namespace: kserve-ci-e2e-test resourceVersion: "22108" uid: 45d446be-c5bb-4512-b301-7036b270391e kind: Event lastTimestamp: "2026-04-16T18:29:26Z" message: Started container sequence-graph-ed24b metadata: creationTimestamp: "2026-04-16T18:29:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:29:26Z" name: sequence-graph-ed24b-7c6b74f4df-gtjnk.18a6e9cc3adf5a94 namespace: kserve-ci-e2e-test resourceVersion: "22132" uid: 965e4f6b-d45b-4f99-ba18-50218ada948f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-ed24b} kind: Pod name: sequence-graph-ed24b-7c6b74f4df-gtjnk namespace: kserve-ci-e2e-test resourceVersion: "22108" uid: 45d446be-c5bb-4512-b301-7036b270391e kind: Event lastTimestamp: "2026-04-16T18:37:40Z" message: Stopping container sequence-graph-ed24b metadata: creationTimestamp: "2026-04-16T18:37:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:37:40Z" name: sequence-graph-ed24b-7c6b74f4df-gtjnk.18a6ea3f4b276d11 namespace: kserve-ci-e2e-test resourceVersion: "25962" uid: aa60b92e-cbd4-4edd-b9fa-36ff6420211a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:37:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-ed24b} kind: Pod name: sequence-graph-ed24b-7c6b74f4df-gtjnk namespace: kserve-ci-e2e-test resourceVersion: "22108" uid: 45d446be-c5bb-4512-b301-7036b270391e kind: Event lastTimestamp: "2026-04-16T18:38:08Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T18:37:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:38:08Z" name: sequence-graph-ed24b-7c6b74f4df-gtjnk.18a6ea3fec8af517 namespace: kserve-ci-e2e-test resourceVersion: "26313" uid: 848b2e30-ac4c-439c-8d1f-d6f3d6311093 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:29:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-ed24b-7c6b74f4df namespace: kserve-ci-e2e-test resourceVersion: "22102" uid: 47c0b963-4844-44e8-8b83-c064b27786df kind: Event lastTimestamp: "2026-04-16T18:29:25Z" message: 'Created pod: sequence-graph-ed24b-7c6b74f4df-gtjnk' metadata: creationTimestamp: "2026-04-16T18:29:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:29:25Z" name: sequence-graph-ed24b-7c6b74f4df.18a6e9cc1c4f95ca namespace: kserve-ci-e2e-test resourceVersion: "22106" uid: e5e1b3c1-2fba-404f-9270-f785bd84cf0c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:29:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-ed24b namespace: kserve-ci-e2e-test resourceVersion: "22100" uid: 324a83aa-fa57-4fc3-a30b-d31168de6697 kind: Event lastTimestamp: "2026-04-16T18:29:25Z" message: Scaled up replica set sequence-graph-ed24b-7c6b74f4df from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:29:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:29:25Z" name: sequence-graph-ed24b.18a6e9cc1b49d0ea namespace: kserve-ci-e2e-test resourceVersion: "22103" uid: 6e459594-faf2-4e80-a89d-f4da90641679 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:29:26Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-ed24b namespace: kserve-ci-e2e-test resourceVersion: "22117" uid: 1df99f25-bc73-4f52-b495-de936188911a kind: Event lastTimestamp: "2026-04-16T18:29:26Z" message: InferenceGraph [sequence-graph-ed24b] is Ready metadata: creationTimestamp: "2026-04-16T18:29:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:29:26Z" name: sequence-graph-ed24b.18a6e9cc2449be0a namespace: kserve-ci-e2e-test resourceVersion: "22119" uid: 85ddeff2-96d5-4fda-8e16-b5dc417f5647 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:29:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-ed24b namespace: kserve-ci-e2e-test resourceVersion: "22101" uid: 211c782c-291a-49b0-9de8-3889eceae74d kind: Event lastTimestamp: "2026-04-16T18:29:40Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:29:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:29:40Z" name: sequence-graph-ed24b.18a6e9cf9a369b13 namespace: kserve-ci-e2e-test resourceVersion: "22246" uid: 3cc04b50-cfe8-40b7-a1d3-92a2f954f3ee reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:29:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-ed24b namespace: kserve-ci-e2e-test resourceVersion: "22101" uid: 211c782c-291a-49b0-9de8-3889eceae74d kind: Event lastTimestamp: "2026-04-16T18:29:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:29:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:29:40Z" name: sequence-graph-ed24b.18a6e9cf9aa089b5 namespace: kserve-ci-e2e-test resourceVersion: "22248" uid: 8fcaaf30-a697-4259-bc4f-0b29e1cd59a1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:29:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-ed24b namespace: kserve-ci-e2e-test resourceVersion: "22247" uid: 211c782c-291a-49b0-9de8-3889eceae74d kind: Event lastTimestamp: "2026-04-16T18:29:55Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:29:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:29:55Z" name: sequence-graph-ed24b.18a6e9d3196569ac namespace: kserve-ci-e2e-test resourceVersion: "22350" uid: 3456e712-868f-4e36-9d28-612a78de5d21 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:29:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-ed24b namespace: kserve-ci-e2e-test resourceVersion: "22247" uid: 211c782c-291a-49b0-9de8-3889eceae74d kind: Event lastTimestamp: "2026-04-16T18:29:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:29:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:29:55Z" name: sequence-graph-ed24b.18a6e9d319c92e3f namespace: kserve-ci-e2e-test resourceVersion: "22352" uid: a8645dd0-d3ec-40aa-8bdb-53ea12365da6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:38:21.062952Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-78581-5cd4b8fdb7-tts6g namespace: kserve-ci-e2e-test resourceVersion: "26433" uid: 5bae1a6a-377e-418b-8998-cd4f92f48afc kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-78581-5cd4b8fdb7-tts6g to ip-10-0-128-59.ec2.internal metadata: creationTimestamp: "2026-04-16T18:38:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:38:21Z" name: splitter-graph-78581-5cd4b8fdb7-tts6g.18a6ea48b6669e82 namespace: kserve-ci-e2e-test resourceVersion: "26438" uid: b1c0a21e-9900-4bfe-8557-c51d5191a2bc reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4b7bbd69-nffhf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:21Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-78581-5cd4b8fdb7-tts6g namespace: kserve-ci-e2e-test resourceVersion: "26437" uid: 5bae1a6a-377e-418b-8998-cd4f92f48afc kind: Event lastTimestamp: "2026-04-16T18:38:21Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-78581-serving-cert" not found' metadata: creationTimestamp: "2026-04-16T18:38:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:38:21Z" name: splitter-graph-78581-5cd4b8fdb7-tts6g.18a6ea48bffa684b namespace: kserve-ci-e2e-test resourceVersion: "26448" uid: 7c78964c-e95b-4014-b00f-2a9926d4e92d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:22Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-78581-5cd4b8fdb7-tts6g namespace: kserve-ci-e2e-test resourceVersion: "26442" uid: 5bae1a6a-377e-418b-8998-cd4f92f48afc kind: Event lastTimestamp: "2026-04-16T18:38:22Z" message: Add eth0 [10.134.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:38:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:38:22Z" name: splitter-graph-78581-5cd4b8fdb7-tts6g.18a6ea48f3761d4c namespace: kserve-ci-e2e-test resourceVersion: "26456" uid: c07662cf-e5e8-42c8-a953-b97444cdcdee reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-78581} kind: Pod name: splitter-graph-78581-5cd4b8fdb7-tts6g namespace: kserve-ci-e2e-test resourceVersion: "26437" uid: 5bae1a6a-377e-418b-8998-cd4f92f48afc kind: Event lastTimestamp: "2026-04-16T18:38:22Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:6060da2148ba78c06d6de96222351f1e5f7d3ada4027ac02142aa0e1928e4f29" already present on machine metadata: creationTimestamp: "2026-04-16T18:38:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:38:22Z" name: splitter-graph-78581-5cd4b8fdb7-tts6g.18a6ea48f4a6e1c5 namespace: kserve-ci-e2e-test resourceVersion: "26458" uid: dc29ba9a-449a-48e0-9073-9d685a678a8b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-78581} kind: Pod name: splitter-graph-78581-5cd4b8fdb7-tts6g namespace: kserve-ci-e2e-test resourceVersion: "26437" uid: 5bae1a6a-377e-418b-8998-cd4f92f48afc kind: Event lastTimestamp: "2026-04-16T18:38:22Z" message: 'Created container: splitter-graph-78581' metadata: creationTimestamp: "2026-04-16T18:38:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:38:22Z" name: splitter-graph-78581-5cd4b8fdb7-tts6g.18a6ea48f78bd87d namespace: kserve-ci-e2e-test resourceVersion: "26460" uid: 87f721a2-c8a2-4b4e-b29e-212fc995ef2d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-78581} kind: Pod name: splitter-graph-78581-5cd4b8fdb7-tts6g namespace: kserve-ci-e2e-test resourceVersion: "26437" uid: 5bae1a6a-377e-418b-8998-cd4f92f48afc kind: Event lastTimestamp: "2026-04-16T18:38:22Z" message: Started container splitter-graph-78581 metadata: creationTimestamp: "2026-04-16T18:38:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:38:22Z" name: splitter-graph-78581-5cd4b8fdb7-tts6g.18a6ea48f7fc4a11 namespace: kserve-ci-e2e-test resourceVersion: "26461" uid: 7f4177c6-6d00-41cd-9952-7832726bbdbf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-78581} kind: Pod name: splitter-graph-78581-5cd4b8fdb7-tts6g namespace: kserve-ci-e2e-test resourceVersion: "26437" uid: 5bae1a6a-377e-418b-8998-cd4f92f48afc kind: Event lastTimestamp: "2026-04-16T18:38:31Z" message: Stopping container splitter-graph-78581 metadata: creationTimestamp: "2026-04-16T18:38:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:38:31Z" name: splitter-graph-78581-5cd4b8fdb7-tts6g.18a6ea4b0bc598b4 namespace: kserve-ci-e2e-test resourceVersion: "26558" uid: 5ed52702-73e3-4d91-b51a-61e7fe6d660f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:38:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-78581} kind: Pod name: splitter-graph-78581-5cd4b8fdb7-tts6g namespace: kserve-ci-e2e-test resourceVersion: "26437" uid: 5bae1a6a-377e-418b-8998-cd4f92f48afc kind: Event lastTimestamp: "2026-04-16T18:38:58Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T18:38:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:38:58Z" name: splitter-graph-78581-5cd4b8fdb7-tts6g.18a6ea4baf5aac83 namespace: kserve-ci-e2e-test resourceVersion: "26940" uid: 579f56fa-066a-4c33-8c1f-585bdad7242b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-78581-5cd4b8fdb7 namespace: kserve-ci-e2e-test resourceVersion: "26430" uid: afa7ab13-974c-47a0-b17c-3f3993f1ca16 kind: Event lastTimestamp: "2026-04-16T18:38:21Z" message: 'Created pod: splitter-graph-78581-5cd4b8fdb7-tts6g' metadata: creationTimestamp: "2026-04-16T18:38:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:38:21Z" name: splitter-graph-78581-5cd4b8fdb7.18a6ea48b591327e namespace: kserve-ci-e2e-test resourceVersion: "26435" uid: 1fa47dc4-c877-460d-94bf-87b0cfbb3bc5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-78581 namespace: kserve-ci-e2e-test resourceVersion: "26428" uid: 6170407f-1278-4996-9c53-3d9c186b6139 kind: Event lastTimestamp: "2026-04-16T18:38:21Z" message: Scaled up replica set splitter-graph-78581-5cd4b8fdb7 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:38:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:38:21Z" name: splitter-graph-78581.18a6ea48b3d399c3 namespace: kserve-ci-e2e-test resourceVersion: "26432" uid: fbe2b614-b7fb-4efb-8309-dd84bf87b350 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:21Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-78581 namespace: kserve-ci-e2e-test resourceVersion: "26444" uid: 899fb979-5d13-4130-91ec-5224b1733787 kind: Event lastTimestamp: "2026-04-16T18:38:21Z" message: InferenceGraph [splitter-graph-78581] is Ready metadata: creationTimestamp: "2026-04-16T18:38:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:38:21Z" name: splitter-graph-78581.18a6ea48baeb1c04 namespace: kserve-ci-e2e-test resourceVersion: "26446" uid: afad3450-a72f-44f5-a660-57287060f173 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:39:41.285490Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-84e06-7f457f648b-sqvbh namespace: kserve-ci-e2e-test resourceVersion: "27302" uid: da4fef60-9248-459d-8b42-5ccd6f77aa0d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-84e06-7f457f648b-sqvbh to ip-10-0-128-59.ec2.internal metadata: creationTimestamp: "2026-04-16T18:39:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:39:41Z" name: splitter-graph-84e06-7f457f648b-sqvbh.18a6ea5b64097392 namespace: kserve-ci-e2e-test resourceVersion: "27307" uid: 51e8659a-dbf1-4811-9c39-58eb440148ca reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4b7bbd69-nffhf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:41Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-84e06-7f457f648b-sqvbh namespace: kserve-ci-e2e-test resourceVersion: "27311" uid: da4fef60-9248-459d-8b42-5ccd6f77aa0d kind: Event lastTimestamp: "2026-04-16T18:39:41Z" message: Add eth0 [10.134.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:39:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:39:41Z" name: splitter-graph-84e06-7f457f648b-sqvbh.18a6ea5b7d219bbd namespace: kserve-ci-e2e-test resourceVersion: "27322" uid: 06e703f2-935a-4200-b699-748fa25e2313 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-84e06} kind: Pod name: splitter-graph-84e06-7f457f648b-sqvbh namespace: kserve-ci-e2e-test resourceVersion: "27304" uid: da4fef60-9248-459d-8b42-5ccd6f77aa0d kind: Event lastTimestamp: "2026-04-16T18:39:41Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:6060da2148ba78c06d6de96222351f1e5f7d3ada4027ac02142aa0e1928e4f29" already present on machine metadata: creationTimestamp: "2026-04-16T18:39:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:39:41Z" name: splitter-graph-84e06-7f457f648b-sqvbh.18a6ea5b7e4748ff namespace: kserve-ci-e2e-test resourceVersion: "27324" uid: e20203c9-d287-4eba-8791-667d3990b41f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-84e06} kind: Pod name: splitter-graph-84e06-7f457f648b-sqvbh namespace: kserve-ci-e2e-test resourceVersion: "27304" uid: da4fef60-9248-459d-8b42-5ccd6f77aa0d kind: Event lastTimestamp: "2026-04-16T18:39:41Z" message: 'Created container: splitter-graph-84e06' metadata: creationTimestamp: "2026-04-16T18:39:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:39:41Z" name: splitter-graph-84e06-7f457f648b-sqvbh.18a6ea5b81f9ea1f namespace: kserve-ci-e2e-test resourceVersion: "27326" uid: 35b33b09-8519-407a-a517-7394b2986398 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-84e06} kind: Pod name: splitter-graph-84e06-7f457f648b-sqvbh namespace: kserve-ci-e2e-test resourceVersion: "27304" uid: da4fef60-9248-459d-8b42-5ccd6f77aa0d kind: Event lastTimestamp: "2026-04-16T18:39:41Z" message: Started container splitter-graph-84e06 metadata: creationTimestamp: "2026-04-16T18:39:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:39:41Z" name: splitter-graph-84e06-7f457f648b-sqvbh.18a6ea5b826c3ef7 namespace: kserve-ci-e2e-test resourceVersion: "27327" uid: aada1022-c8af-4ee6-80da-a5ca552f1850 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:47:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-84e06} kind: Pod name: splitter-graph-84e06-7f457f648b-sqvbh namespace: kserve-ci-e2e-test resourceVersion: "27304" uid: da4fef60-9248-459d-8b42-5ccd6f77aa0d kind: Event lastTimestamp: "2026-04-16T18:47:56Z" message: Stopping container splitter-graph-84e06 metadata: creationTimestamp: "2026-04-16T18:47:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:47:56Z" name: splitter-graph-84e06-7f457f648b-sqvbh.18a6eace94a3d48f namespace: kserve-ci-e2e-test resourceVersion: "30762" uid: aa2f88ec-5f8a-4b11-930a-955f1c5208e8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:47:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-84e06} kind: Pod name: splitter-graph-84e06-7f457f648b-sqvbh namespace: kserve-ci-e2e-test resourceVersion: "27304" uid: da4fef60-9248-459d-8b42-5ccd6f77aa0d kind: Event lastTimestamp: "2026-04-16T18:48:23Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T18:47:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:48:23Z" name: splitter-graph-84e06-7f457f648b-sqvbh.18a6eacf0ea2f472 namespace: kserve-ci-e2e-test resourceVersion: "30987" uid: 9ea59a3e-540c-4ec6-9ef6-21a5b4f07083 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-84e06-7f457f648b namespace: kserve-ci-e2e-test resourceVersion: "27299" uid: 0a9a22a5-9501-4208-bfd9-f3b602068b63 kind: Event lastTimestamp: "2026-04-16T18:39:41Z" message: 'Created pod: splitter-graph-84e06-7f457f648b-sqvbh' metadata: creationTimestamp: "2026-04-16T18:39:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:39:41Z" name: splitter-graph-84e06-7f457f648b.18a6ea5b6399bab3 namespace: kserve-ci-e2e-test resourceVersion: "27303" uid: 65768e3c-6b82-4d05-9b42-a52f9d83d426 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-84e06 namespace: kserve-ci-e2e-test resourceVersion: "27297" uid: 74c646ef-05f2-4ae3-beb3-b55cc797bd54 kind: Event lastTimestamp: "2026-04-16T18:39:41Z" message: Scaled up replica set splitter-graph-84e06-7f457f648b from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:39:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:39:41Z" name: splitter-graph-84e06.18a6ea5b62e87730 namespace: kserve-ci-e2e-test resourceVersion: "27300" uid: 99359a07-8904-426a-9332-e7fd196307ec reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:41Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-84e06 namespace: kserve-ci-e2e-test resourceVersion: "27316" uid: 31ec1527-5bf8-4c04-8946-7bec49c878a4 kind: Event lastTimestamp: "2026-04-16T18:39:41Z" message: InferenceGraph [splitter-graph-84e06] is Ready metadata: creationTimestamp: "2026-04-16T18:39:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:39:41Z" name: splitter-graph-84e06.18a6ea5b6935bab1 namespace: kserve-ci-e2e-test resourceVersion: "27318" uid: 4fa95384-4d6a-45b9-9a53-72b32af57422 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-84e06 namespace: kserve-ci-e2e-test resourceVersion: "27298" uid: 9d527127-e5bd-45c6-85b9-25c88935500a kind: Event lastTimestamp: "2026-04-16T18:39:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:39:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:39:56Z" name: splitter-graph-84e06.18a6ea5ee16eb66b namespace: kserve-ci-e2e-test resourceVersion: "27437" uid: c645003c-1600-4c9a-8801-ba741a164893 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-84e06 namespace: kserve-ci-e2e-test resourceVersion: "27298" uid: 9d527127-e5bd-45c6-85b9-25c88935500a kind: Event lastTimestamp: "2026-04-16T18:39:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:39:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:39:56Z" name: splitter-graph-84e06.18a6ea5ee1d99d91 namespace: kserve-ci-e2e-test resourceVersion: "27439" uid: 65277ef5-13af-4e30-af39-97694173d5a8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:40:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-84e06 namespace: kserve-ci-e2e-test resourceVersion: "27438" uid: 9d527127-e5bd-45c6-85b9-25c88935500a kind: Event lastTimestamp: "2026-04-16T18:40:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:40:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:40:11Z" name: splitter-graph-84e06.18a6ea62609ebdb7 namespace: kserve-ci-e2e-test resourceVersion: "27540" uid: fcc1be92-a64d-4b97-8a28-a151275e27f7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:40:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-84e06 namespace: kserve-ci-e2e-test resourceVersion: "27438" uid: 9d527127-e5bd-45c6-85b9-25c88935500a kind: Event lastTimestamp: "2026-04-16T18:40:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:40:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:40:11Z" name: splitter-graph-84e06.18a6ea6261178012 namespace: kserve-ci-e2e-test resourceVersion: "27542" uid: db58e240-221c-4cd3-820b-d9856aa92ec1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:17:30.579025Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-0403e-predictor-5bfb47d58c-d59t9 namespace: kserve-ci-e2e-test resourceVersion: "16040" uid: 3993f680-7849-4f19-9b81-85dd3c228e72 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-0403e-predictor-5bfb47d58c-d59t9 to ip-10-0-128-59.ec2.internal metadata: creationTimestamp: "2026-04-16T18:17:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:17:30Z" name: success-200-isvc-0403e-predictor-5bfb47d58c-d59t9.18a6e9258fc00f12 namespace: kserve-ci-e2e-test resourceVersion: "16047" uid: 5b752fdc-b580-4380-b51d-b49fd3590b19 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4b7bbd69-nffhf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:17:30Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-0403e-predictor-5bfb47d58c-d59t9 namespace: kserve-ci-e2e-test resourceVersion: "16056" uid: 3993f680-7849-4f19-9b81-85dd3c228e72 kind: Event lastTimestamp: "2026-04-16T18:17:30Z" message: Add eth0 [10.134.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:17:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:17:30Z" name: success-200-isvc-0403e-predictor-5bfb47d58c-d59t9.18a6e92598173fb7 namespace: kserve-ci-e2e-test resourceVersion: "16095" uid: ebf8c2f4-e87a-414b-88db-cf601b64add2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:17:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0403e-predictor-5bfb47d58c-d59t9 namespace: kserve-ci-e2e-test resourceVersion: "16043" uid: 3993f680-7849-4f19-9b81-85dd3c228e72 kind: Event lastTimestamp: "2026-04-16T18:17:30Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1412" already present on machine metadata: creationTimestamp: "2026-04-16T18:17:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:17:30Z" name: success-200-isvc-0403e-predictor-5bfb47d58c-d59t9.18a6e9259acccc26 namespace: kserve-ci-e2e-test resourceVersion: "16102" uid: e6338c86-3c50-4420-8ecf-60eca854ed04 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:17:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0403e-predictor-5bfb47d58c-d59t9 namespace: kserve-ci-e2e-test resourceVersion: "16043" uid: 3993f680-7849-4f19-9b81-85dd3c228e72 kind: Event lastTimestamp: "2026-04-16T18:17:30Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:17:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:17:30Z" name: success-200-isvc-0403e-predictor-5bfb47d58c-d59t9.18a6e9259ef88a10 namespace: kserve-ci-e2e-test resourceVersion: "16121" uid: 488a3b41-3adc-4600-a5f1-71acc4fb430e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:17:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0403e-predictor-5bfb47d58c-d59t9 namespace: kserve-ci-e2e-test resourceVersion: "16043" uid: 3993f680-7849-4f19-9b81-85dd3c228e72 kind: Event lastTimestamp: "2026-04-16T18:17:30Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:17:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:17:30Z" name: success-200-isvc-0403e-predictor-5bfb47d58c-d59t9.18a6e9259f71d554 namespace: kserve-ci-e2e-test resourceVersion: "16123" uid: 67af2a18-6ced-4bc7-b167-bcf2b5e1b7b3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:17:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0403e-predictor-5bfb47d58c-d59t9 namespace: kserve-ci-e2e-test resourceVersion: "16043" uid: 3993f680-7849-4f19-9b81-85dd3c228e72 kind: Event lastTimestamp: "2026-04-16T18:18:12Z" message: 'Readiness probe failed: dial tcp 10.134.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:17:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:18:12Z" name: success-200-isvc-0403e-predictor-5bfb47d58c-d59t9.18a6e925b6702f21 namespace: kserve-ci-e2e-test resourceVersion: "16528" uid: 9c48f449-eb28-4420-b1c2-2d3fe17fee66 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0403e-predictor-5bfb47d58c-d59t9 namespace: kserve-ci-e2e-test resourceVersion: "16043" uid: 3993f680-7849-4f19-9b81-85dd3c228e72 kind: Event lastTimestamp: "2026-04-16T18:26:55Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:26:55Z" name: success-200-isvc-0403e-predictor-5bfb47d58c-d59t9.18a6e9a91995c55a namespace: kserve-ci-e2e-test resourceVersion: "20377" uid: a742d9a7-59b1-41fc-b2b1-5e409bbed29f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:17:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-0403e-predictor-5bfb47d58c namespace: kserve-ci-e2e-test resourceVersion: "16026" uid: 958afb3a-8f3b-4883-b03c-cedb93612c96 kind: Event lastTimestamp: "2026-04-16T18:17:30Z" message: 'Created pod: success-200-isvc-0403e-predictor-5bfb47d58c-d59t9' metadata: creationTimestamp: "2026-04-16T18:17:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:17:30Z" name: success-200-isvc-0403e-predictor-5bfb47d58c.18a6e9258f5dc68d namespace: kserve-ci-e2e-test resourceVersion: "16045" uid: d6152334-af2a-41be-afd4-ad9d03389b82 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:17:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-0403e-predictor namespace: kserve-ci-e2e-test resourceVersion: "16023" uid: f360f8ab-01df-4093-a4ff-ff776ea64a34 kind: Event lastTimestamp: "2026-04-16T18:17:30Z" message: Scaled up replica set success-200-isvc-0403e-predictor-5bfb47d58c from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:17:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:17:30Z" name: success-200-isvc-0403e-predictor.18a6e9258d14e916 namespace: kserve-ci-e2e-test resourceVersion: "16028" uid: 090a58d5-dfc7-4ca6-8c70-bc286285bb19 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:17:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0403e-predictor namespace: kserve-ci-e2e-test resourceVersion: "16025" uid: 7fe2724e-97c7-44e4-bc02-1e992606cc4c kind: Event lastTimestamp: "2026-04-16T18:18:00Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:17:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:18:00Z" name: success-200-isvc-0403e-predictor.18a6e9290bfe5c78 namespace: kserve-ci-e2e-test resourceVersion: "16405" uid: 24c3b8da-26af-41b5-bbcc-3a32f6b02e2e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:17:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0403e-predictor namespace: kserve-ci-e2e-test resourceVersion: "16025" uid: 7fe2724e-97c7-44e4-bc02-1e992606cc4c kind: Event lastTimestamp: "2026-04-16T18:18:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:17:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:18:00Z" name: success-200-isvc-0403e-predictor.18a6e9290c4f05af namespace: kserve-ci-e2e-test resourceVersion: "16406" uid: f5d9a6b3-e2cc-4bc6-8c6e-9fc5f7562a2a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:18:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0403e-predictor namespace: kserve-ci-e2e-test resourceVersion: "16263" uid: 7fe2724e-97c7-44e4-bc02-1e992606cc4c kind: Event lastTimestamp: "2026-04-16T18:18:45Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:18:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:18:45Z" name: success-200-isvc-0403e-predictor.18a6e9300a9599b2 namespace: kserve-ci-e2e-test resourceVersion: "16840" uid: eb7c9f77-0660-40fb-9f1c-e44c75d88be0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:18:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0403e-predictor namespace: kserve-ci-e2e-test resourceVersion: "16263" uid: 7fe2724e-97c7-44e4-bc02-1e992606cc4c kind: Event lastTimestamp: "2026-04-16T18:18:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:18:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:18:45Z" name: success-200-isvc-0403e-predictor.18a6e9300b147d69 namespace: kserve-ci-e2e-test resourceVersion: "16841" uid: 36fff7fa-5d9c-4452-8b41-250f51aa175d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:17:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0403e namespace: kserve-ci-e2e-test resourceVersion: "16016" uid: 5dee3f22-8e23-48d4-8812-4d1fef5a0eda kind: Event lastTimestamp: "2026-04-16T18:18:22Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:17:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:18:22Z" name: success-200-isvc-0403e.18a6e9258d1818c1 namespace: kserve-ci-e2e-test resourceVersion: "16625" uid: 53e0856a-726b-408c-b152-44938b618a56 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0403e namespace: kserve-ci-e2e-test resourceVersion: "16623" uid: 5dee3f22-8e23-48d4-8812-4d1fef5a0eda kind: Event lastTimestamp: "2026-04-16T18:18:22Z" message: InferenceService [success-200-isvc-0403e] is Ready metadata: creationTimestamp: "2026-04-16T18:18:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:18:22Z" name: success-200-isvc-0403e.18a6e9319af90347 namespace: kserve-ci-e2e-test resourceVersion: "16624" uid: 6f32c7be-d38a-4f80-96bd-2ee9bf4752d6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:17:00.663111Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-04c3d-predictor-6c8787dd55-x9kf9 namespace: kserve-ci-e2e-test resourceVersion: "15613" uid: 61350c93-dd52-4dca-b279-01c128b89c20 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-04c3d-predictor-6c8787dd55-x9kf9 to ip-10-0-128-59.ec2.internal metadata: creationTimestamp: "2026-04-16T18:17:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:17:00Z" name: success-200-isvc-04c3d-predictor-6c8787dd55-x9kf9.18a6e91e989f7174 namespace: kserve-ci-e2e-test resourceVersion: "15627" uid: e1971dc6-11b1-43bc-84c1-fc06a032ceac reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4b7bbd69-nffhf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:17:00Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-04c3d-predictor-6c8787dd55-x9kf9 namespace: kserve-ci-e2e-test resourceVersion: "15631" uid: 61350c93-dd52-4dca-b279-01c128b89c20 kind: Event lastTimestamp: "2026-04-16T18:17:00Z" message: Add eth0 [10.134.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:17:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:17:00Z" name: success-200-isvc-04c3d-predictor-6c8787dd55-x9kf9.18a6e91ea0e00b22 namespace: kserve-ci-e2e-test resourceVersion: "15659" uid: 4ec21a22-44a1-49bc-b1e3-0a0e39fec74c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:17:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-04c3d-predictor-6c8787dd55-x9kf9 namespace: kserve-ci-e2e-test resourceVersion: "15615" uid: 61350c93-dd52-4dca-b279-01c128b89c20 kind: Event lastTimestamp: "2026-04-16T18:17:00Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1412" already present on machine metadata: creationTimestamp: "2026-04-16T18:17:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:17:00Z" name: success-200-isvc-04c3d-predictor-6c8787dd55-x9kf9.18a6e91ea22795c8 namespace: kserve-ci-e2e-test resourceVersion: "15665" uid: ee8c237e-e07f-4fd7-a100-6aec66d1e60e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:17:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-04c3d-predictor-6c8787dd55-x9kf9 namespace: kserve-ci-e2e-test resourceVersion: "15615" uid: 61350c93-dd52-4dca-b279-01c128b89c20 kind: Event lastTimestamp: "2026-04-16T18:17:00Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:17:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:17:00Z" name: success-200-isvc-04c3d-predictor-6c8787dd55-x9kf9.18a6e91ea7cbab01 namespace: kserve-ci-e2e-test resourceVersion: "15669" uid: 0b42a57c-7020-4109-8a64-3218b3e1ae9b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:17:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-04c3d-predictor-6c8787dd55-x9kf9 namespace: kserve-ci-e2e-test resourceVersion: "15615" uid: 61350c93-dd52-4dca-b279-01c128b89c20 kind: Event lastTimestamp: "2026-04-16T18:17:00Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:17:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:17:00Z" name: success-200-isvc-04c3d-predictor-6c8787dd55-x9kf9.18a6e91ea847860f namespace: kserve-ci-e2e-test resourceVersion: "15670" uid: 0b976cd2-2d81-448a-af0d-26f5668999eb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:17:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-04c3d-predictor-6c8787dd55-x9kf9 namespace: kserve-ci-e2e-test resourceVersion: "15615" uid: 61350c93-dd52-4dca-b279-01c128b89c20 kind: Event lastTimestamp: "2026-04-16T18:17:42Z" message: 'Readiness probe failed: dial tcp 10.134.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:17:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:17:42Z" name: success-200-isvc-04c3d-predictor-6c8787dd55-x9kf9.18a6e91eb4a76ea1 namespace: kserve-ci-e2e-test resourceVersion: "16236" uid: 730c0987-68e3-46fb-84c0-f69684fd490d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-04c3d-predictor-6c8787dd55-x9kf9 namespace: kserve-ci-e2e-test resourceVersion: "15615" uid: 61350c93-dd52-4dca-b279-01c128b89c20 kind: Event lastTimestamp: "2026-04-16T18:26:25Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:26:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:26:25Z" name: success-200-isvc-04c3d-predictor-6c8787dd55-x9kf9.18a6e9a2262d3f34 namespace: kserve-ci-e2e-test resourceVersion: "20020" uid: 1c6c5345-513b-4eef-a65f-ac8f25d2a401 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:17:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-04c3d-predictor-6c8787dd55 namespace: kserve-ci-e2e-test resourceVersion: "15601" uid: 725983d6-e8f9-49ef-896d-d4dc1f63111a kind: Event lastTimestamp: "2026-04-16T18:17:00Z" message: 'Created pod: success-200-isvc-04c3d-predictor-6c8787dd55-x9kf9' metadata: creationTimestamp: "2026-04-16T18:17:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:17:00Z" name: success-200-isvc-04c3d-predictor-6c8787dd55.18a6e91e96cc3dc4 namespace: kserve-ci-e2e-test resourceVersion: "15616" uid: 52f2126f-dc23-4571-b431-662a0b045194 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:17:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-04c3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15600" uid: 492fadf0-0341-4247-a168-5ccec2ad7300 kind: Event lastTimestamp: "2026-04-16T18:17:00Z" message: Scaled up replica set success-200-isvc-04c3d-predictor-6c8787dd55 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:17:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:17:00Z" name: success-200-isvc-04c3d-predictor.18a6e91e950c9085 namespace: kserve-ci-e2e-test resourceVersion: "15604" uid: 8abdc17d-c384-49ea-b1e1-94bf0aa1d766 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T18:17:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-04c3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15603" uid: 922b9d56-c89b-4b5a-9c14-6776e478ab48 kind: Event lastTimestamp: "2026-04-16T18:18:00Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:17:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:18:00Z" name: success-200-isvc-04c3d-predictor.18a6e922140afcb5 namespace: kserve-ci-e2e-test resourceVersion: "16408" uid: db1d23c6-4ff6-4e97-93f2-9c6f99123e7b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T18:17:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-04c3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15603" uid: 922b9d56-c89b-4b5a-9c14-6776e478ab48 kind: Event lastTimestamp: "2026-04-16T18:18:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:17:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:18:00Z" name: success-200-isvc-04c3d-predictor.18a6e922148ab00b namespace: kserve-ci-e2e-test resourceVersion: "16409" uid: fecca6f4-cced-407f-89e6-f28f676ac875 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:17:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-04c3d namespace: kserve-ci-e2e-test resourceVersion: "15580" uid: 84e89921-922d-4253-8bac-545c81ac28cd kind: Event lastTimestamp: "2026-04-16T18:17:52Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:17:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:17:52Z" name: success-200-isvc-04c3d.18a6e91e9546beb1 namespace: kserve-ci-e2e-test resourceVersion: "16334" uid: 6a78028e-b8c9-4465-ab6d-50a5640ad82e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:17:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-04c3d namespace: kserve-ci-e2e-test resourceVersion: "16330" uid: 84e89921-922d-4253-8bac-545c81ac28cd kind: Event lastTimestamp: "2026-04-16T18:17:52Z" message: InferenceService [success-200-isvc-04c3d] is Ready metadata: creationTimestamp: "2026-04-16T18:17:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:17:52Z" name: success-200-isvc-04c3d.18a6e92a98fd6575 namespace: kserve-ci-e2e-test resourceVersion: "16332" uid: a590cc5f-ca3b-4d98-9b81-1d011dc7fe27 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:15:30.238440Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-390f9-predictor-5bdd998c9c-dxsnc namespace: kserve-ci-e2e-test resourceVersion: "14560" uid: a28f8fdd-ca6f-4fd5-9f84-66cc20786923 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-390f9-predictor-5bdd998c9c-dxsnc to ip-10-0-128-59.ec2.internal metadata: creationTimestamp: "2026-04-16T18:15:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:15:30Z" name: success-200-isvc-390f9-predictor-5bdd998c9c-dxsnc.18a6e9098ae47973 namespace: kserve-ci-e2e-test resourceVersion: "14565" uid: 4cd669d6-2458-47d4-b88b-4855a4a2da17 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4b7bbd69-nffhf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:15:30Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-390f9-predictor-5bdd998c9c-dxsnc namespace: kserve-ci-e2e-test resourceVersion: "14569" uid: a28f8fdd-ca6f-4fd5-9f84-66cc20786923 kind: Event lastTimestamp: "2026-04-16T18:15:30Z" message: Add eth0 [10.134.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:15:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:15:30Z" name: success-200-isvc-390f9-predictor-5bdd998c9c-dxsnc.18a6e90992a1026b namespace: kserve-ci-e2e-test resourceVersion: "14578" uid: 77a929d6-3e61-4443-ba46-2c28eeeb4a58 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:15:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-390f9-predictor-5bdd998c9c-dxsnc namespace: kserve-ci-e2e-test resourceVersion: "14562" uid: a28f8fdd-ca6f-4fd5-9f84-66cc20786923 kind: Event lastTimestamp: "2026-04-16T18:15:30Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1412" metadata: creationTimestamp: "2026-04-16T18:15:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:15:30Z" name: success-200-isvc-390f9-predictor-5bdd998c9c-dxsnc.18a6e909958c7867 namespace: kserve-ci-e2e-test resourceVersion: "14584" uid: 105944e0-f04b-48b4-9550-a7b4c39e5386 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:15:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-390f9-predictor-5bdd998c9c-dxsnc namespace: kserve-ci-e2e-test resourceVersion: "14562" uid: a28f8fdd-ca6f-4fd5-9f84-66cc20786923 kind: Event lastTimestamp: "2026-04-16T18:15:42Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1412" in 11.622s (11.622s including waiting). Image size: 1332223599 bytes.' metadata: creationTimestamp: "2026-04-16T18:15:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:15:42Z" name: success-200-isvc-390f9-predictor-5bdd998c9c-dxsnc.18a6e90c4a553b4c namespace: kserve-ci-e2e-test resourceVersion: "14809" uid: ddd2485c-b898-43de-bdf6-82454d8af8ce reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:15:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-390f9-predictor-5bdd998c9c-dxsnc namespace: kserve-ci-e2e-test resourceVersion: "14562" uid: a28f8fdd-ca6f-4fd5-9f84-66cc20786923 kind: Event lastTimestamp: "2026-04-16T18:15:42Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:15:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:15:42Z" name: success-200-isvc-390f9-predictor-5bdd998c9c-dxsnc.18a6e90c4ea66363 namespace: kserve-ci-e2e-test resourceVersion: "14810" uid: a1aceaba-5dc2-4c57-b342-f8cb32121647 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:15:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-390f9-predictor-5bdd998c9c-dxsnc namespace: kserve-ci-e2e-test resourceVersion: "14562" uid: a28f8fdd-ca6f-4fd5-9f84-66cc20786923 kind: Event lastTimestamp: "2026-04-16T18:15:42Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:15:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:15:42Z" name: success-200-isvc-390f9-predictor-5bdd998c9c-dxsnc.18a6e90c4f04d8a5 namespace: kserve-ci-e2e-test resourceVersion: "14812" uid: 81bae484-ae2c-4065-aebc-00fcf130b461 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:15:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-390f9-predictor-5bdd998c9c-dxsnc namespace: kserve-ci-e2e-test resourceVersion: "14562" uid: a28f8fdd-ca6f-4fd5-9f84-66cc20786923 kind: Event lastTimestamp: "2026-04-16T18:16:23Z" message: 'Readiness probe failed: dial tcp 10.134.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:15:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:16:23Z" name: success-200-isvc-390f9-predictor-5bdd998c9c-dxsnc.18a6e90c7e3802e7 namespace: kserve-ci-e2e-test resourceVersion: "15200" uid: 2017725b-9ece-4a59-b08b-0aa6c5a70142 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:17:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-390f9-predictor-5bdd998c9c-dxsnc namespace: kserve-ci-e2e-test resourceVersion: "14562" uid: a28f8fdd-ca6f-4fd5-9f84-66cc20786923 kind: Event lastTimestamp: "2026-04-16T18:17:00Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:17:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:17:00Z" name: success-200-isvc-390f9-predictor-5bdd998c9c-dxsnc.18a6e91e94a3e460 namespace: kserve-ci-e2e-test resourceVersion: "15602" uid: 69ace7de-c870-4e8d-be51-bf6b98362928 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:17:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-390f9-predictor-5bdd998c9c-dxsnc namespace: kserve-ci-e2e-test resourceVersion: "14562" uid: a28f8fdd-ca6f-4fd5-9f84-66cc20786923 kind: Event lastTimestamp: "2026-04-16T18:17:04Z" message: 'Readiness probe failed: dial tcp 10.134.0.21:8080: i/o timeout' metadata: creationTimestamp: "2026-04-16T18:17:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:17:04Z" name: success-200-isvc-390f9-predictor-5bdd998c9c-dxsnc.18a6e91f9600cc80 namespace: kserve-ci-e2e-test resourceVersion: "15726" uid: 8083e1b4-3cf0-42aa-b7a2-f3976d6442af reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:15:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-390f9-predictor-5bdd998c9c namespace: kserve-ci-e2e-test resourceVersion: "14551" uid: 89fdb1d4-ad60-4a68-838d-804dc7ef2109 kind: Event lastTimestamp: "2026-04-16T18:15:30Z" message: 'Created pod: success-200-isvc-390f9-predictor-5bdd998c9c-dxsnc' metadata: creationTimestamp: "2026-04-16T18:15:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:15:30Z" name: success-200-isvc-390f9-predictor-5bdd998c9c.18a6e9098a501b83 namespace: kserve-ci-e2e-test resourceVersion: "14563" uid: 98a4ddbe-905c-4699-8787-c8dc6dd2e031 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:15:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-390f9-predictor namespace: kserve-ci-e2e-test resourceVersion: "14548" uid: d980234c-8b51-4683-b834-6117dabb1a19 kind: Event lastTimestamp: "2026-04-16T18:15:30Z" message: Scaled up replica set success-200-isvc-390f9-predictor-5bdd998c9c from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:15:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:15:30Z" name: success-200-isvc-390f9-predictor.18a6e90984a28129 namespace: kserve-ci-e2e-test resourceVersion: "14553" uid: eb8ff889-9ae3-4a06-bd6d-4af5aa77bf74 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:15:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-390f9-predictor namespace: kserve-ci-e2e-test resourceVersion: "14559" uid: 17d2e8b5-5735-475a-8f70-33d6669fd274 kind: Event lastTimestamp: "2026-04-16T18:16:00Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:15:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:16:00Z" name: success-200-isvc-390f9-predictor.18a6e90d09d0e4b0 namespace: kserve-ci-e2e-test resourceVersion: "15002" uid: 28a5a102-0f17-442d-afbd-6c8c9843aaf5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:15:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-390f9-predictor namespace: kserve-ci-e2e-test resourceVersion: "14559" uid: 17d2e8b5-5735-475a-8f70-33d6669fd274 kind: Event lastTimestamp: "2026-04-16T18:16:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:15:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:16:00Z" name: success-200-isvc-390f9-predictor.18a6e90d0a2716f1 namespace: kserve-ci-e2e-test resourceVersion: "15003" uid: 8e4d7b6f-7a02-4e1d-bf1a-4a9e5635259e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T18:16:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-390f9-predictor namespace: kserve-ci-e2e-test resourceVersion: "14847" uid: 17d2e8b5-5735-475a-8f70-33d6669fd274 kind: Event lastTimestamp: "2026-04-16T18:17:00Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:16:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:17:00Z" name: success-200-isvc-390f9-predictor.18a6e9140866733f namespace: kserve-ci-e2e-test resourceVersion: "15555" uid: b278443c-5427-4ff3-889d-bbfb3fa717b5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T18:16:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-390f9-predictor namespace: kserve-ci-e2e-test resourceVersion: "14847" uid: 17d2e8b5-5735-475a-8f70-33d6669fd274 kind: Event lastTimestamp: "2026-04-16T18:17:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:16:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:17:00Z" name: success-200-isvc-390f9-predictor.18a6e91408e3deed namespace: kserve-ci-e2e-test resourceVersion: "15556" uid: b7458584-a169-417a-8b37-ca402aa06da2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T18:15:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-390f9 namespace: kserve-ci-e2e-test resourceVersion: "14539" uid: 314404f9-8485-4f2d-91f8-b3c4f6756597 kind: Event lastTimestamp: "2026-04-16T18:16:34Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:15:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:16:34Z" name: success-200-isvc-390f9.18a6e9098a131a68 namespace: kserve-ci-e2e-test resourceVersion: "15296" uid: 19573a86-1263-437f-a384-dcb61596db16 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:15:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-390f9 namespace: kserve-ci-e2e-test resourceVersion: "14539" uid: 314404f9-8485-4f2d-91f8-b3c4f6756597 kind: Event lastTimestamp: "2026-04-16T18:15:30Z" message: 'Failed to update status for InferenceService "success-200-isvc-390f9": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-390f9": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:15:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:15:30Z" name: success-200-isvc-390f9.18a6e9098aa9193a namespace: kserve-ci-e2e-test resourceVersion: "14566" uid: 07e64e9c-a427-49a3-aa49-2550d882062a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:15:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-390f9 namespace: kserve-ci-e2e-test resourceVersion: "14539" uid: 314404f9-8485-4f2d-91f8-b3c4f6756597 kind: Event lastTimestamp: "2026-04-16T18:15:30Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-390f9": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:15:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:15:30Z" name: success-200-isvc-390f9.18a6e9098aa97f93 namespace: kserve-ci-e2e-test resourceVersion: "14568" uid: 4429edd2-e33b-4460-b72b-a36938ff0660 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:16:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-390f9 namespace: kserve-ci-e2e-test resourceVersion: "15294" uid: 314404f9-8485-4f2d-91f8-b3c4f6756597 kind: Event lastTimestamp: "2026-04-16T18:16:33Z" message: InferenceService [success-200-isvc-390f9] is Ready metadata: creationTimestamp: "2026-04-16T18:16:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:16:33Z" name: success-200-isvc-390f9.18a6e91862f451a0 namespace: kserve-ci-e2e-test resourceVersion: "15295" uid: a265ddf9-bae7-40d1-8f02-daa0689b28be reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:26:55.544778Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-516af-predictor-7dc5688b5-z5fmq namespace: kserve-ci-e2e-test resourceVersion: "20373" uid: e07116fe-c239-43f6-beac-013fd89bbf35 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-516af-predictor-7dc5688b5-z5fmq to ip-10-0-128-59.ec2.internal metadata: creationTimestamp: "2026-04-16T18:26:55Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:26:55Z" name: success-200-isvc-516af-predictor-7dc5688b5-z5fmq.18a6e9a91a5558b7 namespace: kserve-ci-e2e-test resourceVersion: "20380" uid: 8f9e38d3-e5ca-48eb-befd-725b09116b7c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4b7bbd69-nffhf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:55Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-516af-predictor-7dc5688b5-z5fmq namespace: kserve-ci-e2e-test resourceVersion: "20384" uid: e07116fe-c239-43f6-beac-013fd89bbf35 kind: Event lastTimestamp: "2026-04-16T18:26:55Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:26:55Z" name: success-200-isvc-516af-predictor-7dc5688b5-z5fmq.18a6e9a925be7733 namespace: kserve-ci-e2e-test resourceVersion: "20434" uid: feec52ae-6cf5-43cc-94d6-ef5901e497de reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-516af-predictor-7dc5688b5-z5fmq namespace: kserve-ci-e2e-test resourceVersion: "20375" uid: e07116fe-c239-43f6-beac-013fd89bbf35 kind: Event lastTimestamp: "2026-04-16T18:26:55Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1412" already present on machine metadata: creationTimestamp: "2026-04-16T18:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:26:55Z" name: success-200-isvc-516af-predictor-7dc5688b5-z5fmq.18a6e9a926db45d9 namespace: kserve-ci-e2e-test resourceVersion: "20436" uid: 96e4eabc-f7cf-4167-9d8e-6804475a0e6c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-516af-predictor-7dc5688b5-z5fmq namespace: kserve-ci-e2e-test resourceVersion: "20375" uid: e07116fe-c239-43f6-beac-013fd89bbf35 kind: Event lastTimestamp: "2026-04-16T18:26:55Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:26:55Z" name: success-200-isvc-516af-predictor-7dc5688b5-z5fmq.18a6e9a92a7e4e46 namespace: kserve-ci-e2e-test resourceVersion: "20441" uid: f42cd4e8-b16d-4af8-a54a-9eac815a558a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-516af-predictor-7dc5688b5-z5fmq namespace: kserve-ci-e2e-test resourceVersion: "20375" uid: e07116fe-c239-43f6-beac-013fd89bbf35 kind: Event lastTimestamp: "2026-04-16T18:26:55Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:26:55Z" name: success-200-isvc-516af-predictor-7dc5688b5-z5fmq.18a6e9a92b0718c6 namespace: kserve-ci-e2e-test resourceVersion: "20444" uid: 33690ded-5d00-4202-81bf-2477e7f850c9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T18:26:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-516af-predictor-7dc5688b5-z5fmq namespace: kserve-ci-e2e-test resourceVersion: "20375" uid: e07116fe-c239-43f6-beac-013fd89bbf35 kind: Event lastTimestamp: "2026-04-16T18:28:16Z" message: 'Readiness probe failed: dial tcp 10.134.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:28:16Z" name: success-200-isvc-516af-predictor-7dc5688b5-z5fmq.18a6e9a92c0c4e34 namespace: kserve-ci-e2e-test resourceVersion: "21473" uid: a085520f-46f6-40b4-bafb-caeecefb96c9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-516af-predictor-7dc5688b5-z5fmq namespace: kserve-ci-e2e-test resourceVersion: "20375" uid: e07116fe-c239-43f6-beac-013fd89bbf35 kind: Event lastTimestamp: "2026-04-16T18:28:15Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:28:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:28:15Z" name: success-200-isvc-516af-predictor-7dc5688b5-z5fmq.18a6e9bbca507c74 namespace: kserve-ci-e2e-test resourceVersion: "21391" uid: adb9ae38-afff-4678-9de6-69a9b0659012 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:55Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-516af-predictor-7dc5688b5 namespace: kserve-ci-e2e-test resourceVersion: "20360" uid: dc8918c0-b33f-4df0-b921-df3ce3ccf981 kind: Event lastTimestamp: "2026-04-16T18:26:55Z" message: 'Created pod: success-200-isvc-516af-predictor-7dc5688b5-z5fmq' metadata: creationTimestamp: "2026-04-16T18:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:26:55Z" name: success-200-isvc-516af-predictor-7dc5688b5.18a6e9a9198f8a21 namespace: kserve-ci-e2e-test resourceVersion: "20376" uid: f9e46fcd-2bfa-4a6c-a3e7-96412f979a0d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:55Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-516af-predictor namespace: kserve-ci-e2e-test resourceVersion: "20354" uid: 9218f523-111a-4e82-8e26-9248858f17fa kind: Event lastTimestamp: "2026-04-16T18:26:55Z" message: Scaled up replica set success-200-isvc-516af-predictor-7dc5688b5 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:26:55Z" name: success-200-isvc-516af-predictor.18a6e9a9183600f7 namespace: kserve-ci-e2e-test resourceVersion: "20365" uid: 887869f7-8fd5-4160-94a5-4622f8fc54de reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:27:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-516af-predictor namespace: kserve-ci-e2e-test resourceVersion: "20358" uid: e6c00ead-aa77-427d-bc5c-54d7b76d1aeb kind: Event lastTimestamp: "2026-04-16T18:27:25Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:27:25Z" name: success-200-isvc-516af-predictor.18a6e9ac972c1675 namespace: kserve-ci-e2e-test resourceVersion: "20712" uid: 8ca15e47-1d4c-4ab5-8070-73ce0945a9f9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:27:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-516af-predictor namespace: kserve-ci-e2e-test resourceVersion: "20358" uid: e6c00ead-aa77-427d-bc5c-54d7b76d1aeb kind: Event lastTimestamp: "2026-04-16T18:27:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:27:25Z" name: success-200-isvc-516af-predictor.18a6e9ac97a56caa namespace: kserve-ci-e2e-test resourceVersion: "20713" uid: 30b1d5f0-3882-44ab-bd14-0bb3d769ee32 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:27:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-516af-predictor namespace: kserve-ci-e2e-test resourceVersion: "20576" uid: e6c00ead-aa77-427d-bc5c-54d7b76d1aeb kind: Event lastTimestamp: "2026-04-16T18:28:10Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:27:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:28:10Z" name: success-200-isvc-516af-predictor.18a6e9b395cfeba6 namespace: kserve-ci-e2e-test resourceVersion: "21296" uid: 4b0719d2-168d-481a-b57a-fdd7f027422a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:27:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-516af-predictor namespace: kserve-ci-e2e-test resourceVersion: "20576" uid: e6c00ead-aa77-427d-bc5c-54d7b76d1aeb kind: Event lastTimestamp: "2026-04-16T18:28:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:27:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:28:10Z" name: success-200-isvc-516af-predictor.18a6e9b3963f3855 namespace: kserve-ci-e2e-test resourceVersion: "21297" uid: 993d4776-8f10-4628-874b-07406fe9c2a3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:26:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-516af namespace: kserve-ci-e2e-test resourceVersion: "20346" uid: f4ad618d-7c56-47e6-a23d-56ecba855106 kind: Event lastTimestamp: "2026-04-16T18:27:47Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:27:47Z" name: success-200-isvc-516af.18a6e9a91814708c namespace: kserve-ci-e2e-test resourceVersion: "21066" uid: 8bd2844e-e122-4033-9c55-ba9757a9b71f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-516af namespace: kserve-ci-e2e-test resourceVersion: "20346" uid: f4ad618d-7c56-47e6-a23d-56ecba855106 kind: Event lastTimestamp: "2026-04-16T18:26:55Z" message: 'Failed to update status for InferenceService "success-200-isvc-516af": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-516af": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:26:55Z" name: success-200-isvc-516af.18a6e9a918dbfdca namespace: kserve-ci-e2e-test resourceVersion: "20371" uid: 8e2a677b-1944-4146-9a4a-8adbc4bd37d4 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-516af namespace: kserve-ci-e2e-test resourceVersion: "20346" uid: f4ad618d-7c56-47e6-a23d-56ecba855106 kind: Event lastTimestamp: "2026-04-16T18:26:55Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-516af": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:26:55Z" name: success-200-isvc-516af.18a6e9a918dc5dab namespace: kserve-ci-e2e-test resourceVersion: "20374" uid: 07235090-f26f-42e6-91f1-d51c4163d612 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-516af namespace: kserve-ci-e2e-test resourceVersion: "21057" uid: f4ad618d-7c56-47e6-a23d-56ecba855106 kind: Event lastTimestamp: "2026-04-16T18:27:46Z" message: InferenceService [success-200-isvc-516af] is Ready metadata: creationTimestamp: "2026-04-16T18:27:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:27:47Z" name: success-200-isvc-516af.18a6e9b512dda657 namespace: kserve-ci-e2e-test resourceVersion: "21064" uid: 76796462-e558-47e0-ae8f-3d6b92f4b7eb reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:37:10.924993Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-78581-predictor-d6d9bf857-m87d6 namespace: kserve-ci-e2e-test resourceVersion: "25650" uid: 996efd22-cdee-405a-8852-b2952981236a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-78581-predictor-d6d9bf857-m87d6 to ip-10-0-128-59.ec2.internal metadata: creationTimestamp: "2026-04-16T18:37:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:37:10Z" name: success-200-isvc-78581-predictor-d6d9bf857-m87d6.18a6ea3861da4e14 namespace: kserve-ci-e2e-test resourceVersion: "25665" uid: 14cc9e84-61d4-43dc-b179-4481008e57fd reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4b7bbd69-nffhf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:11Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-78581-predictor-d6d9bf857-m87d6 namespace: kserve-ci-e2e-test resourceVersion: "25675" uid: 996efd22-cdee-405a-8852-b2952981236a kind: Event lastTimestamp: "2026-04-16T18:37:11Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:37:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:37:11Z" name: success-200-isvc-78581-predictor-d6d9bf857-m87d6.18a6ea3869af5d95 namespace: kserve-ci-e2e-test resourceVersion: "25697" uid: 089d2d89-b096-4905-83cc-194d397bce5e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-78581-predictor-d6d9bf857-m87d6 namespace: kserve-ci-e2e-test resourceVersion: "25653" uid: 996efd22-cdee-405a-8852-b2952981236a kind: Event lastTimestamp: "2026-04-16T18:37:11Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1412" already present on machine metadata: creationTimestamp: "2026-04-16T18:37:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:37:11Z" name: success-200-isvc-78581-predictor-d6d9bf857-m87d6.18a6ea386ad7ec43 namespace: kserve-ci-e2e-test resourceVersion: "25699" uid: 7b95674a-7b1d-42b7-98fd-fbcd268139f2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-78581-predictor-d6d9bf857-m87d6 namespace: kserve-ci-e2e-test resourceVersion: "25653" uid: 996efd22-cdee-405a-8852-b2952981236a kind: Event lastTimestamp: "2026-04-16T18:37:11Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:37:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:37:11Z" name: success-200-isvc-78581-predictor-d6d9bf857-m87d6.18a6ea3873197fe3 namespace: kserve-ci-e2e-test resourceVersion: "25708" uid: 02bea83a-7b57-46f4-9e44-4cca79d1f53c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-78581-predictor-d6d9bf857-m87d6 namespace: kserve-ci-e2e-test resourceVersion: "25653" uid: 996efd22-cdee-405a-8852-b2952981236a kind: Event lastTimestamp: "2026-04-16T18:37:11Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:37:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:37:11Z" name: success-200-isvc-78581-predictor-d6d9bf857-m87d6.18a6ea38739075f7 namespace: kserve-ci-e2e-test resourceVersion: "25709" uid: 7a005685-cda0-476c-bac6-2e4c6e311079 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T18:37:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-78581-predictor-d6d9bf857-m87d6 namespace: kserve-ci-e2e-test resourceVersion: "25653" uid: 996efd22-cdee-405a-8852-b2952981236a kind: Event lastTimestamp: "2026-04-16T18:38:32Z" message: 'Readiness probe failed: dial tcp 10.134.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:37:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:38:32Z" name: success-200-isvc-78581-predictor-d6d9bf857-m87d6.18a6ea388ac80488 namespace: kserve-ci-e2e-test resourceVersion: "26676" uid: 5ad098cd-0e1a-4255-a5e8-39cec3aa6fce reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-78581-predictor-d6d9bf857-m87d6 namespace: kserve-ci-e2e-test resourceVersion: "25653" uid: 996efd22-cdee-405a-8852-b2952981236a kind: Event lastTimestamp: "2026-04-16T18:38:31Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:38:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:38:31Z" name: success-200-isvc-78581-predictor-d6d9bf857-m87d6.18a6ea4b12b3b9a4 namespace: kserve-ci-e2e-test resourceVersion: "26589" uid: b42c7ab5-4906-489e-8017-fd9b1797de9d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-78581-predictor-d6d9bf857 namespace: kserve-ci-e2e-test resourceVersion: "25635" uid: 8948fae2-3e73-4823-849a-6f2a81d0e9fa kind: Event lastTimestamp: "2026-04-16T18:37:10Z" message: 'Created pod: success-200-isvc-78581-predictor-d6d9bf857-m87d6' metadata: creationTimestamp: "2026-04-16T18:37:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:37:10Z" name: success-200-isvc-78581-predictor-d6d9bf857.18a6ea38616fb608 namespace: kserve-ci-e2e-test resourceVersion: "25654" uid: 93dc45f6-b5ee-4e5f-a793-c586682b4ff4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-78581-predictor namespace: kserve-ci-e2e-test resourceVersion: "25630" uid: 0d75a01b-d975-4dd5-bccf-9108d9549f0e kind: Event lastTimestamp: "2026-04-16T18:37:10Z" message: Scaled up replica set success-200-isvc-78581-predictor-d6d9bf857 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:37:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:37:10Z" name: success-200-isvc-78581-predictor.18a6ea385e271f3c namespace: kserve-ci-e2e-test resourceVersion: "25637" uid: 4e621e53-ad8c-4872-bdbe-b28e2bb43f93 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:37:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-78581-predictor namespace: kserve-ci-e2e-test resourceVersion: "25634" uid: dc8fa144-4422-490a-82a0-9102f4e3877c kind: Event lastTimestamp: "2026-04-16T18:37:40Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:37:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:37:40Z" name: success-200-isvc-78581-predictor.18a6ea3bdcd63ab7 namespace: kserve-ci-e2e-test resourceVersion: "26044" uid: 82b4ba1a-f0f6-47ab-a761-de59d45a47ff reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:37:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-78581-predictor namespace: kserve-ci-e2e-test resourceVersion: "25634" uid: dc8fa144-4422-490a-82a0-9102f4e3877c kind: Event lastTimestamp: "2026-04-16T18:37:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:37:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:37:40Z" name: success-200-isvc-78581-predictor.18a6ea3bdd24fa41 namespace: kserve-ci-e2e-test resourceVersion: "26045" uid: a4ef20de-0dc0-4a23-9257-a8b2e9f4edb3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:37:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-78581-predictor namespace: kserve-ci-e2e-test resourceVersion: "25840" uid: dc8fa144-4422-490a-82a0-9102f4e3877c kind: Event lastTimestamp: "2026-04-16T18:38:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:37:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:38:25Z" name: success-200-isvc-78581-predictor.18a6ea42daa09ef7 namespace: kserve-ci-e2e-test resourceVersion: "26497" uid: e72f6bb9-4645-45e4-8566-944f4f42925c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:37:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-78581-predictor namespace: kserve-ci-e2e-test resourceVersion: "25840" uid: dc8fa144-4422-490a-82a0-9102f4e3877c kind: Event lastTimestamp: "2026-04-16T18:38:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:37:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:38:25Z" name: success-200-isvc-78581-predictor.18a6ea42db0494de namespace: kserve-ci-e2e-test resourceVersion: "26498" uid: 8a12104f-55c2-4fc9-a29f-b24fb2347897 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:37:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-78581 namespace: kserve-ci-e2e-test resourceVersion: "25617" uid: bc4bf1db-2332-4c4b-a259-50ced197670a kind: Event lastTimestamp: "2026-04-16T18:38:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:37:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:38:02Z" name: success-200-isvc-78581.18a6ea385e1b2a30 namespace: kserve-ci-e2e-test resourceVersion: "26265" uid: 130162fc-7f51-4bee-ad81-259bb739e1fa reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-78581 namespace: kserve-ci-e2e-test resourceVersion: "25617" uid: bc4bf1db-2332-4c4b-a259-50ced197670a kind: Event lastTimestamp: "2026-04-16T18:37:10Z" message: 'Failed to update status for InferenceService "success-200-isvc-78581": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-78581": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:37:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:37:10Z" name: success-200-isvc-78581.18a6ea385eac3c9b namespace: kserve-ci-e2e-test resourceVersion: "25640" uid: 06352655-921b-47ef-badf-f5f13272bddc reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-78581 namespace: kserve-ci-e2e-test resourceVersion: "25617" uid: bc4bf1db-2332-4c4b-a259-50ced197670a kind: Event lastTimestamp: "2026-04-16T18:37:10Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-78581": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:37:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:37:10Z" name: success-200-isvc-78581.18a6ea385eaca1f7 namespace: kserve-ci-e2e-test resourceVersion: "25643" uid: 27649a4d-f3f6-4474-abca-e71aaba0238b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-78581 namespace: kserve-ci-e2e-test resourceVersion: "26262" uid: bc4bf1db-2332-4c4b-a259-50ced197670a kind: Event lastTimestamp: "2026-04-16T18:38:02Z" message: InferenceService [success-200-isvc-78581] is Ready metadata: creationTimestamp: "2026-04-16T18:38:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:38:02Z" name: success-200-isvc-78581.18a6ea446ecd350a namespace: kserve-ci-e2e-test resourceVersion: "26263" uid: d21b386a-932e-458e-a346-93033cbfa5a3 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:38:31.209670Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-84e06-predictor-579c9cdd97-z77lb namespace: kserve-ci-e2e-test resourceVersion: "26587" uid: 367f033e-09d9-4af4-bf1c-d7d5d49ba20e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-84e06-predictor-579c9cdd97-z77lb to ip-10-0-128-59.ec2.internal metadata: creationTimestamp: "2026-04-16T18:38:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:38:31Z" name: success-200-isvc-84e06-predictor-579c9cdd97-z77lb.18a6ea4b13314398 namespace: kserve-ci-e2e-test resourceVersion: "26594" uid: 9a221fa6-8469-44d7-94c0-02506b3ca0b1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4b7bbd69-nffhf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:31Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-84e06-predictor-579c9cdd97-z77lb namespace: kserve-ci-e2e-test resourceVersion: "26598" uid: 367f033e-09d9-4af4-bf1c-d7d5d49ba20e kind: Event lastTimestamp: "2026-04-16T18:38:31Z" message: Add eth0 [10.134.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:38:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:38:31Z" name: success-200-isvc-84e06-predictor-579c9cdd97-z77lb.18a6ea4b1b1259ae namespace: kserve-ci-e2e-test resourceVersion: "26640" uid: 88021ae1-48ff-4edb-94c5-0d43c6b13980 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-84e06-predictor-579c9cdd97-z77lb namespace: kserve-ci-e2e-test resourceVersion: "26590" uid: 367f033e-09d9-4af4-bf1c-d7d5d49ba20e kind: Event lastTimestamp: "2026-04-16T18:38:31Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1412" already present on machine metadata: creationTimestamp: "2026-04-16T18:38:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:38:31Z" name: success-200-isvc-84e06-predictor-579c9cdd97-z77lb.18a6ea4b1c59ddd0 namespace: kserve-ci-e2e-test resourceVersion: "26644" uid: 05fba349-46a8-447e-a9dc-eabc2907fe8c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-84e06-predictor-579c9cdd97-z77lb namespace: kserve-ci-e2e-test resourceVersion: "26590" uid: 367f033e-09d9-4af4-bf1c-d7d5d49ba20e kind: Event lastTimestamp: "2026-04-16T18:38:31Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:38:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:38:31Z" name: success-200-isvc-84e06-predictor-579c9cdd97-z77lb.18a6ea4b201ceefb namespace: kserve-ci-e2e-test resourceVersion: "26648" uid: f8d724d9-1c0e-4509-ace3-8c40de930b05 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-84e06-predictor-579c9cdd97-z77lb namespace: kserve-ci-e2e-test resourceVersion: "26590" uid: 367f033e-09d9-4af4-bf1c-d7d5d49ba20e kind: Event lastTimestamp: "2026-04-16T18:38:31Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:38:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:38:31Z" name: success-200-isvc-84e06-predictor-579c9cdd97-z77lb.18a6ea4b20b24c4d namespace: kserve-ci-e2e-test resourceVersion: "26649" uid: ab54f496-6663-433a-b50a-8884b213554a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:38:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-84e06-predictor-579c9cdd97-z77lb namespace: kserve-ci-e2e-test resourceVersion: "26590" uid: 367f033e-09d9-4af4-bf1c-d7d5d49ba20e kind: Event lastTimestamp: "2026-04-16T18:39:12Z" message: 'Readiness probe failed: dial tcp 10.134.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:38:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:39:12Z" name: success-200-isvc-84e06-predictor-579c9cdd97-z77lb.18a6ea4b3957b671 namespace: kserve-ci-e2e-test resourceVersion: "27060" uid: 7952ddb8-1afd-4cb1-a7b9-f3a17e7381e4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:47:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-84e06-predictor-579c9cdd97-z77lb namespace: kserve-ci-e2e-test resourceVersion: "26590" uid: 367f033e-09d9-4af4-bf1c-d7d5d49ba20e kind: Event lastTimestamp: "2026-04-16T18:47:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:47:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:47:56Z" name: success-200-isvc-84e06-predictor-579c9cdd97-z77lb.18a6eace9a18ba35 namespace: kserve-ci-e2e-test resourceVersion: "30775" uid: 640be05e-fc53-4ca3-8755-83e8c09790dc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-84e06-predictor-579c9cdd97 namespace: kserve-ci-e2e-test resourceVersion: "26574" uid: 62d070a8-2288-4583-b311-72c7d54ae1d0 kind: Event lastTimestamp: "2026-04-16T18:38:31Z" message: 'Created pod: success-200-isvc-84e06-predictor-579c9cdd97-z77lb' metadata: creationTimestamp: "2026-04-16T18:38:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:38:31Z" name: success-200-isvc-84e06-predictor-579c9cdd97.18a6ea4b12bbcc4e namespace: kserve-ci-e2e-test resourceVersion: "26591" uid: c869002d-82d9-42b5-ba8e-20257fb5f343 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-84e06-predictor namespace: kserve-ci-e2e-test resourceVersion: "26568" uid: 992baac4-7c30-4808-8771-f7246b2731e4 kind: Event lastTimestamp: "2026-04-16T18:38:31Z" message: Scaled up replica set success-200-isvc-84e06-predictor-579c9cdd97 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:38:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:38:31Z" name: success-200-isvc-84e06-predictor.18a6ea4b113f74ee namespace: kserve-ci-e2e-test resourceVersion: "26577" uid: 7135684a-628c-4384-8f5a-3b4d1f33599d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:38:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-84e06-predictor namespace: kserve-ci-e2e-test resourceVersion: "26573" uid: ab244a5f-ad07-447a-ba00-0108f95a411e kind: Event lastTimestamp: "2026-04-16T18:39:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:38:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:39:01Z" name: success-200-isvc-84e06-predictor.18a6ea4e8fe84e0f namespace: kserve-ci-e2e-test resourceVersion: "26960" uid: a6becc9b-25d3-46fd-b82c-4893221024b7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:38:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-84e06-predictor namespace: kserve-ci-e2e-test resourceVersion: "26573" uid: ab244a5f-ad07-447a-ba00-0108f95a411e kind: Event lastTimestamp: "2026-04-16T18:39:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:38:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:39:01Z" name: success-200-isvc-84e06-predictor.18a6ea4e9053ea7f namespace: kserve-ci-e2e-test resourceVersion: "26961" uid: 6a4d5a32-4822-4e10-b561-bd7636949ef4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:39:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-84e06-predictor namespace: kserve-ci-e2e-test resourceVersion: "26808" uid: ab244a5f-ad07-447a-ba00-0108f95a411e kind: Event lastTimestamp: "2026-04-16T18:39:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:39:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:39:46Z" name: success-200-isvc-84e06-predictor.18a6ea558deba3fd namespace: kserve-ci-e2e-test resourceVersion: "27360" uid: 9a968ee0-72e7-4c4c-95e8-cb7a7812bbc6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:39:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-84e06-predictor namespace: kserve-ci-e2e-test resourceVersion: "26808" uid: ab244a5f-ad07-447a-ba00-0108f95a411e kind: Event lastTimestamp: "2026-04-16T18:39:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:39:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:39:46Z" name: success-200-isvc-84e06-predictor.18a6ea558e5f925f namespace: kserve-ci-e2e-test resourceVersion: "27361" uid: 066220b5-0714-4368-9a91-9b21fd87bae8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:38:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-84e06 namespace: kserve-ci-e2e-test resourceVersion: "26561" uid: b7d15ec5-56f1-4349-91a3-8ae6a6b113d9 kind: Event lastTimestamp: "2026-04-16T18:39:22Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:38:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:39:22Z" name: success-200-isvc-84e06.18a6ea4b113f84a6 namespace: kserve-ci-e2e-test resourceVersion: "27146" uid: 52cc3d54-93a3-4580-bdd9-236557e5c370 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-84e06 namespace: kserve-ci-e2e-test resourceVersion: "26561" uid: b7d15ec5-56f1-4349-91a3-8ae6a6b113d9 kind: Event lastTimestamp: "2026-04-16T18:38:31Z" message: 'Failed to update status for InferenceService "success-200-isvc-84e06": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-84e06": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:38:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:38:31Z" name: success-200-isvc-84e06.18a6ea4b11dc15e1 namespace: kserve-ci-e2e-test resourceVersion: "26584" uid: 5e6405a0-2201-4e43-8e8b-f35940c421b8 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-84e06 namespace: kserve-ci-e2e-test resourceVersion: "26561" uid: b7d15ec5-56f1-4349-91a3-8ae6a6b113d9 kind: Event lastTimestamp: "2026-04-16T18:38:31Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-84e06": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:38:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:38:31Z" name: success-200-isvc-84e06.18a6ea4b11dc7c8f namespace: kserve-ci-e2e-test resourceVersion: "26588" uid: f86e7f49-bed1-4dbd-96c7-4a3b7d4d3489 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-84e06 namespace: kserve-ci-e2e-test resourceVersion: "27143" uid: b7d15ec5-56f1-4349-91a3-8ae6a6b113d9 kind: Event lastTimestamp: "2026-04-16T18:39:22Z" message: InferenceService [success-200-isvc-84e06] is Ready metadata: creationTimestamp: "2026-04-16T18:39:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:39:22Z" name: success-200-isvc-84e06.18a6ea571d3fe02c namespace: kserve-ci-e2e-test resourceVersion: "27144" uid: b5308696-b505-42c4-ba2a-37df2c2d98a1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:37:40.770669Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-89f53-predictor-646dcc4bbf-8hgx4 namespace: kserve-ci-e2e-test resourceVersion: "25996" uid: 28d7194e-4cc1-46d4-befc-1faef9286d01 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-89f53-predictor-646dcc4bbf-8hgx4 to ip-10-0-128-59.ec2.internal metadata: creationTimestamp: "2026-04-16T18:37:40Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:37:40Z" name: success-200-isvc-89f53-predictor-646dcc4bbf-8hgx4.18a6ea3f54cb241a namespace: kserve-ci-e2e-test resourceVersion: "26000" uid: 7c29309a-4846-4cd0-a87f-60599b1ef71b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4b7bbd69-nffhf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:40Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-89f53-predictor-646dcc4bbf-8hgx4 namespace: kserve-ci-e2e-test resourceVersion: "26006" uid: 28d7194e-4cc1-46d4-befc-1faef9286d01 kind: Event lastTimestamp: "2026-04-16T18:37:40Z" message: Add eth0 [10.134.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:37:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:37:40Z" name: success-200-isvc-89f53-predictor-646dcc4bbf-8hgx4.18a6ea3f5f2cb26d namespace: kserve-ci-e2e-test resourceVersion: "26049" uid: 2b689270-c625-4655-bd60-43c8916aa7ef reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-89f53-predictor-646dcc4bbf-8hgx4 namespace: kserve-ci-e2e-test resourceVersion: "25997" uid: 28d7194e-4cc1-46d4-befc-1faef9286d01 kind: Event lastTimestamp: "2026-04-16T18:37:40Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1412" already present on machine metadata: creationTimestamp: "2026-04-16T18:37:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:37:40Z" name: success-200-isvc-89f53-predictor-646dcc4bbf-8hgx4.18a6ea3f6050442f namespace: kserve-ci-e2e-test resourceVersion: "26051" uid: da67f2b7-b1e3-42ee-a31f-bb590e172c52 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-89f53-predictor-646dcc4bbf-8hgx4 namespace: kserve-ci-e2e-test resourceVersion: "25997" uid: 28d7194e-4cc1-46d4-befc-1faef9286d01 kind: Event lastTimestamp: "2026-04-16T18:37:41Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:37:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:37:41Z" name: success-200-isvc-89f53-predictor-646dcc4bbf-8hgx4.18a6ea3f638fdd64 namespace: kserve-ci-e2e-test resourceVersion: "26058" uid: 93d0f02b-5b67-446d-8c22-d07598378150 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-89f53-predictor-646dcc4bbf-8hgx4 namespace: kserve-ci-e2e-test resourceVersion: "25997" uid: 28d7194e-4cc1-46d4-befc-1faef9286d01 kind: Event lastTimestamp: "2026-04-16T18:37:41Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:37:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:37:41Z" name: success-200-isvc-89f53-predictor-646dcc4bbf-8hgx4.18a6ea3f640309ee namespace: kserve-ci-e2e-test resourceVersion: "26059" uid: 5d58d9e7-0b49-427a-b021-67be60abad22 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T18:37:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-89f53-predictor-646dcc4bbf-8hgx4 namespace: kserve-ci-e2e-test resourceVersion: "25997" uid: 28d7194e-4cc1-46d4-befc-1faef9286d01 kind: Event lastTimestamp: "2026-04-16T18:55:12Z" message: 'Readiness probe failed: dial tcp 10.134.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:37:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:55:12Z" name: success-200-isvc-89f53-predictor-646dcc4bbf-8hgx4.18a6ea3f8c54b426 namespace: kserve-ci-e2e-test resourceVersion: "33838" uid: ffde2044-c12d-4daf-b1ef-dc8c88b54b9c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:55:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-89f53-predictor-646dcc4bbf-8hgx4 namespace: kserve-ci-e2e-test resourceVersion: "25997" uid: 28d7194e-4cc1-46d4-befc-1faef9286d01 kind: Event lastTimestamp: "2026-04-16T18:55:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:55:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:55:10Z" name: success-200-isvc-89f53-predictor-646dcc4bbf-8hgx4.18a6eb33b6583f9d namespace: kserve-ci-e2e-test resourceVersion: "33776" uid: 31e7f8fb-709f-4958-a2ec-dbd95fc25713 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:40Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-89f53-predictor-646dcc4bbf namespace: kserve-ci-e2e-test resourceVersion: "25985" uid: b97b890d-3fef-4273-9396-10e43ebce0c9 kind: Event lastTimestamp: "2026-04-16T18:37:40Z" message: 'Created pod: success-200-isvc-89f53-predictor-646dcc4bbf-8hgx4' metadata: creationTimestamp: "2026-04-16T18:37:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:37:40Z" name: success-200-isvc-89f53-predictor-646dcc4bbf.18a6ea3f5474c3b0 namespace: kserve-ci-e2e-test resourceVersion: "25998" uid: f6d7ae2f-f3ad-42f0-a7ea-2425412236f9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:40Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-89f53-predictor namespace: kserve-ci-e2e-test resourceVersion: "25975" uid: 280f1bf1-541f-4d0e-ad1e-3466622f0bc6 kind: Event lastTimestamp: "2026-04-16T18:37:40Z" message: Scaled up replica set success-200-isvc-89f53-predictor-646dcc4bbf from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:37:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:37:40Z" name: success-200-isvc-89f53-predictor.18a6ea3f528acb37 namespace: kserve-ci-e2e-test resourceVersion: "25988" uid: d02ec9ff-8ee4-425b-b0b5-fe1dd647c40f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:37:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-89f53-predictor namespace: kserve-ci-e2e-test resourceVersion: "25978" uid: 74ad9297-462f-4857-b322-6716d0e41ae0 kind: Event lastTimestamp: "2026-04-16T18:38:10Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:37:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:38:10Z" name: success-200-isvc-89f53-predictor.18a6ea42d077fcde namespace: kserve-ci-e2e-test resourceVersion: "26334" uid: 42fc823c-07b2-4611-93cd-ed2c10275d47 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:37:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-89f53-predictor namespace: kserve-ci-e2e-test resourceVersion: "25978" uid: 74ad9297-462f-4857-b322-6716d0e41ae0 kind: Event lastTimestamp: "2026-04-16T18:38:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:37:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:38:10Z" name: success-200-isvc-89f53-predictor.18a6ea42d0df55ed namespace: kserve-ci-e2e-test resourceVersion: "26335" uid: 283e065d-0bc4-4342-a0f7-21e1bfc88231 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T18:38:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-89f53-predictor namespace: kserve-ci-e2e-test resourceVersion: "26200" uid: 74ad9297-462f-4857-b322-6716d0e41ae0 kind: Event lastTimestamp: "2026-04-16T18:39:10Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:38:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:39:10Z" name: success-200-isvc-89f53-predictor.18a6ea49cefca5f3 namespace: kserve-ci-e2e-test resourceVersion: "27041" uid: 5f962c59-701e-4524-8c19-01baf73a289e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-16T18:38:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-89f53-predictor namespace: kserve-ci-e2e-test resourceVersion: "26200" uid: 74ad9297-462f-4857-b322-6716d0e41ae0 kind: Event lastTimestamp: "2026-04-16T18:39:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:38:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:39:10Z" name: success-200-isvc-89f53-predictor.18a6ea49cf545762 namespace: kserve-ci-e2e-test resourceVersion: "27042" uid: 7e130640-27ef-4cc5-9f42-c052d73325c7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:37:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-89f53 namespace: kserve-ci-e2e-test resourceVersion: "25965" uid: 1a082bca-96e7-4ddd-b768-b98285b9ba5c kind: Event lastTimestamp: "2026-04-16T18:38:32Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:37:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:38:32Z" name: success-200-isvc-89f53.18a6ea3f519e14aa namespace: kserve-ci-e2e-test resourceVersion: "26685" uid: 845e4c6f-5c83-43c3-8245-a81c6fa2540f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-89f53 namespace: kserve-ci-e2e-test resourceVersion: "25965" uid: 1a082bca-96e7-4ddd-b768-b98285b9ba5c kind: Event lastTimestamp: "2026-04-16T18:37:40Z" message: 'Failed to update status for InferenceService "success-200-isvc-89f53": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-89f53": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:37:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:37:40Z" name: success-200-isvc-89f53.18a6ea3f52a8131b namespace: kserve-ci-e2e-test resourceVersion: "25990" uid: 1eaa00e9-9641-42f9-a712-4601c00cdc29 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-89f53 namespace: kserve-ci-e2e-test resourceVersion: "25965" uid: 1a082bca-96e7-4ddd-b768-b98285b9ba5c kind: Event lastTimestamp: "2026-04-16T18:37:40Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-89f53": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:37:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:37:40Z" name: success-200-isvc-89f53.18a6ea3f52a87dfc namespace: kserve-ci-e2e-test resourceVersion: "25992" uid: 83648926-bd4a-40e2-8daa-1968f8977b12 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-89f53 namespace: kserve-ci-e2e-test resourceVersion: "26683" uid: 1a082bca-96e7-4ddd-b768-b98285b9ba5c kind: Event lastTimestamp: "2026-04-16T18:38:32Z" message: InferenceService [success-200-isvc-89f53] is Ready metadata: creationTimestamp: "2026-04-16T18:38:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:38:32Z" name: success-200-isvc-89f53.18a6ea4b7032c327 namespace: kserve-ci-e2e-test resourceVersion: "26684" uid: a7246e2e-c00f-4444-aa1a-193c7ae2ab9c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:26:25.741472Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ccdbb-predictor-697459d95f-7kx24 namespace: kserve-ci-e2e-test resourceVersion: "20028" uid: a50cbc4a-a789-49ff-827a-96113af18a02 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-ccdbb-predictor-697459d95f-7kx24 to ip-10-0-128-59.ec2.internal metadata: creationTimestamp: "2026-04-16T18:26:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:26:25Z" name: success-200-isvc-ccdbb-predictor-697459d95f-7kx24.18a6e9a229eafaf5 namespace: kserve-ci-e2e-test resourceVersion: "20046" uid: 50a2c594-8273-4a73-98b0-356a17298a03 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4b7bbd69-nffhf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:25Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ccdbb-predictor-697459d95f-7kx24 namespace: kserve-ci-e2e-test resourceVersion: "20056" uid: a50cbc4a-a789-49ff-827a-96113af18a02 kind: Event lastTimestamp: "2026-04-16T18:26:25Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:26:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:26:25Z" name: success-200-isvc-ccdbb-predictor-697459d95f-7kx24.18a6e9a231d772ad namespace: kserve-ci-e2e-test resourceVersion: "20076" uid: 15b4cacc-3bd8-4032-9216-b25995abeed5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ccdbb-predictor-697459d95f-7kx24 namespace: kserve-ci-e2e-test resourceVersion: "20034" uid: a50cbc4a-a789-49ff-827a-96113af18a02 kind: Event lastTimestamp: "2026-04-16T18:26:25Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1412" already present on machine metadata: creationTimestamp: "2026-04-16T18:26:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:26:25Z" name: success-200-isvc-ccdbb-predictor-697459d95f-7kx24.18a6e9a2334802dc namespace: kserve-ci-e2e-test resourceVersion: "20078" uid: 54938af0-da29-4170-97d9-c244d5dda38f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ccdbb-predictor-697459d95f-7kx24 namespace: kserve-ci-e2e-test resourceVersion: "20034" uid: a50cbc4a-a789-49ff-827a-96113af18a02 kind: Event lastTimestamp: "2026-04-16T18:26:26Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:26:26Z" name: success-200-isvc-ccdbb-predictor-697459d95f-7kx24.18a6e9a23a47223a namespace: kserve-ci-e2e-test resourceVersion: "20085" uid: 4391971e-fb9a-418d-865c-4e4f8b8c16a7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ccdbb-predictor-697459d95f-7kx24 namespace: kserve-ci-e2e-test resourceVersion: "20034" uid: a50cbc4a-a789-49ff-827a-96113af18a02 kind: Event lastTimestamp: "2026-04-16T18:26:26Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:26:26Z" name: success-200-isvc-ccdbb-predictor-697459d95f-7kx24.18a6e9a23ac9098a namespace: kserve-ci-e2e-test resourceVersion: "20087" uid: 4370237b-ff57-4880-9960-155896180904 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-16T18:26:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ccdbb-predictor-697459d95f-7kx24 namespace: kserve-ci-e2e-test resourceVersion: "20034" uid: a50cbc4a-a789-49ff-827a-96113af18a02 kind: Event lastTimestamp: "2026-04-16T18:27:47Z" message: 'Readiness probe failed: dial tcp 10.134.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:27:47Z" name: success-200-isvc-ccdbb-predictor-697459d95f-7kx24.18a6e9a2665575cc namespace: kserve-ci-e2e-test resourceVersion: "21076" uid: be3f5352-dde1-4b7e-9a1e-dbc7757cc866 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ccdbb-predictor-697459d95f-7kx24 namespace: kserve-ci-e2e-test resourceVersion: "20034" uid: a50cbc4a-a789-49ff-827a-96113af18a02 kind: Event lastTimestamp: "2026-04-16T18:27:45Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:27:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:27:46Z" name: success-200-isvc-ccdbb-predictor-697459d95f-7kx24.18a6e9b4d99c06c5 namespace: kserve-ci-e2e-test resourceVersion: "20975" uid: 1a32be9f-80f5-46b2-8178-3fe0e69920a0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-ccdbb-predictor-697459d95f namespace: kserve-ci-e2e-test resourceVersion: "20010" uid: 379bcecb-fc5c-4f95-9a60-305b24f6365b kind: Event lastTimestamp: "2026-04-16T18:26:25Z" message: 'Created pod: success-200-isvc-ccdbb-predictor-697459d95f-7kx24' metadata: creationTimestamp: "2026-04-16T18:26:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:26:25Z" name: success-200-isvc-ccdbb-predictor-697459d95f.18a6e9a2296b080a namespace: kserve-ci-e2e-test resourceVersion: "20033" uid: 6f4dceaf-4815-4cdc-b84d-d733b2f28594 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-ccdbb-predictor namespace: kserve-ci-e2e-test resourceVersion: "20004" uid: 379ef78d-2ab8-4bde-8839-fab9b88ed3d3 kind: Event lastTimestamp: "2026-04-16T18:26:25Z" message: Scaled up replica set success-200-isvc-ccdbb-predictor-697459d95f from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:26:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:26:25Z" name: success-200-isvc-ccdbb-predictor.18a6e9a22540da74 namespace: kserve-ci-e2e-test resourceVersion: "20014" uid: e00e24f8-de7d-47ab-91b8-ed8f83febdb2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:26:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ccdbb-predictor namespace: kserve-ci-e2e-test resourceVersion: "20012" uid: 4b167aa8-d2c7-42f8-b181-ea0319b22c9b kind: Event lastTimestamp: "2026-04-16T18:26:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:26:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:26:55Z" name: success-200-isvc-ccdbb-predictor.18a6e9a5a4743057 namespace: kserve-ci-e2e-test resourceVersion: "20426" uid: e9d1f252-3425-4cf2-91ea-54976832a04c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:26:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ccdbb-predictor namespace: kserve-ci-e2e-test resourceVersion: "20012" uid: 4b167aa8-d2c7-42f8-b181-ea0319b22c9b kind: Event lastTimestamp: "2026-04-16T18:26:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:26:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:26:55Z" name: success-200-isvc-ccdbb-predictor.18a6e9a5a4e15734 namespace: kserve-ci-e2e-test resourceVersion: "20427" uid: 6f037d0d-be5d-48f6-8fb6-2e4d777333a6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:27:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ccdbb-predictor namespace: kserve-ci-e2e-test resourceVersion: "20218" uid: 4b167aa8-d2c7-42f8-b181-ea0319b22c9b kind: Event lastTimestamp: "2026-04-16T18:27:40Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:27:40Z" name: success-200-isvc-ccdbb-predictor.18a6e9aca26c7c82 namespace: kserve-ci-e2e-test resourceVersion: "20878" uid: 144cdedb-94c6-48d9-b4c3-1735c44f36aa reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:27:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ccdbb-predictor namespace: kserve-ci-e2e-test resourceVersion: "20218" uid: 4b167aa8-d2c7-42f8-b181-ea0319b22c9b kind: Event lastTimestamp: "2026-04-16T18:27:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:27:40Z" name: success-200-isvc-ccdbb-predictor.18a6e9aca2d59af7 namespace: kserve-ci-e2e-test resourceVersion: "20879" uid: 938d95ec-c24e-487d-a4d5-149f64192ff4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T18:26:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ccdbb namespace: kserve-ci-e2e-test resourceVersion: "19994" uid: 1e660bed-064d-4128-b39a-6cde23298fab kind: Event lastTimestamp: "2026-04-16T18:27:17Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:26:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:27:17Z" name: success-200-isvc-ccdbb.18a6e9a2255ce5cf namespace: kserve-ci-e2e-test resourceVersion: "20655" uid: 1e9c82ae-1235-4434-b7de-2e14a7eae276 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ccdbb namespace: kserve-ci-e2e-test resourceVersion: "19994" uid: 1e660bed-064d-4128-b39a-6cde23298fab kind: Event lastTimestamp: "2026-04-16T18:26:25Z" message: 'Failed to update status for InferenceService "success-200-isvc-ccdbb": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-ccdbb": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:26:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:26:25Z" name: success-200-isvc-ccdbb.18a6e9a225cacc97 namespace: kserve-ci-e2e-test resourceVersion: "20019" uid: 9f8cba20-0a0b-4172-a813-d75ede91e114 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ccdbb namespace: kserve-ci-e2e-test resourceVersion: "19994" uid: 1e660bed-064d-4128-b39a-6cde23298fab kind: Event lastTimestamp: "2026-04-16T18:26:25Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-ccdbb": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:26:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:26:25Z" name: success-200-isvc-ccdbb.18a6e9a225cb1081 namespace: kserve-ci-e2e-test resourceVersion: "20021" uid: 3cb8cdd7-4c76-4b72-ac0f-fa533714fa39 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ccdbb namespace: kserve-ci-e2e-test resourceVersion: "20653" uid: 1e660bed-064d-4128-b39a-6cde23298fab kind: Event lastTimestamp: "2026-04-16T18:27:17Z" message: InferenceService [success-200-isvc-ccdbb] is Ready metadata: creationTimestamp: "2026-04-16T18:27:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:27:17Z" name: success-200-isvc-ccdbb.18a6e9ae4acc48f9 namespace: kserve-ci-e2e-test resourceVersion: "20654" uid: 460b2942-5f49-4437-a2be-c5377a8e53e3 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:27:46.017108Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-df772-predictor-875748d7f-jdp2w namespace: kserve-ci-e2e-test resourceVersion: "20968" uid: 28dbc4e6-4622-4258-91ad-f18f9f5275ff kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-df772-predictor-875748d7f-jdp2w to ip-10-0-128-59.ec2.internal metadata: creationTimestamp: "2026-04-16T18:27:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:27:46Z" name: success-200-isvc-df772-predictor-875748d7f-jdp2w.18a6e9b4dab7f98e namespace: kserve-ci-e2e-test resourceVersion: "20976" uid: f66d667b-13d1-475f-93b0-b73cbeb8397b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4b7bbd69-nffhf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:46Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-df772-predictor-875748d7f-jdp2w namespace: kserve-ci-e2e-test resourceVersion: "20982" uid: 28dbc4e6-4622-4258-91ad-f18f9f5275ff kind: Event lastTimestamp: "2026-04-16T18:27:46Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:27:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:27:46Z" name: success-200-isvc-df772-predictor-875748d7f-jdp2w.18a6e9b4e2dd3f18 namespace: kserve-ci-e2e-test resourceVersion: "21026" uid: c87e5600-9c37-4bdc-9f74-f902b83802fa reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-df772-predictor-875748d7f-jdp2w namespace: kserve-ci-e2e-test resourceVersion: "20971" uid: 28dbc4e6-4622-4258-91ad-f18f9f5275ff kind: Event lastTimestamp: "2026-04-16T18:27:46Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1412" already present on machine metadata: creationTimestamp: "2026-04-16T18:27:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:27:46Z" name: success-200-isvc-df772-predictor-875748d7f-jdp2w.18a6e9b4e408fe85 namespace: kserve-ci-e2e-test resourceVersion: "21028" uid: 3a544b30-06fa-417c-929e-3be6605946da reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-df772-predictor-875748d7f-jdp2w namespace: kserve-ci-e2e-test resourceVersion: "20971" uid: 28dbc4e6-4622-4258-91ad-f18f9f5275ff kind: Event lastTimestamp: "2026-04-16T18:27:46Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:27:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:27:46Z" name: success-200-isvc-df772-predictor-875748d7f-jdp2w.18a6e9b4ec85ee76 namespace: kserve-ci-e2e-test resourceVersion: "21038" uid: 710e6eea-258e-4fee-9b07-deb9464962f8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-df772-predictor-875748d7f-jdp2w namespace: kserve-ci-e2e-test resourceVersion: "20971" uid: 28dbc4e6-4622-4258-91ad-f18f9f5275ff kind: Event lastTimestamp: "2026-04-16T18:27:46Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:27:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:27:46Z" name: success-200-isvc-df772-predictor-875748d7f-jdp2w.18a6e9b4ecff53a3 namespace: kserve-ci-e2e-test resourceVersion: "21039" uid: 2faf92a2-430c-43ef-a64b-635ec0ca4b94 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:27:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-df772-predictor-875748d7f-jdp2w namespace: kserve-ci-e2e-test resourceVersion: "20971" uid: 28dbc4e6-4622-4258-91ad-f18f9f5275ff kind: Event lastTimestamp: "2026-04-16T18:28:28Z" message: 'Readiness probe failed: dial tcp 10.134.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:27:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:28:28Z" name: success-200-isvc-df772-predictor-875748d7f-jdp2w.18a6e9b515393e5f namespace: kserve-ci-e2e-test resourceVersion: "21570" uid: d99df1e4-2cfc-4741-88a2-a947dc4f4778 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-df772-predictor-875748d7f-jdp2w namespace: kserve-ci-e2e-test resourceVersion: "20971" uid: 28dbc4e6-4622-4258-91ad-f18f9f5275ff kind: Event lastTimestamp: "2026-04-16T18:37:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:37:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:37:10Z" name: success-200-isvc-df772-predictor-875748d7f-jdp2w.18a6ea385f466c50 namespace: kserve-ci-e2e-test resourceVersion: "25644" uid: b4c6c93c-15cb-4331-a702-9d05424c6dd9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:46Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-df772-predictor-875748d7f namespace: kserve-ci-e2e-test resourceVersion: "20955" uid: b1fdfd8d-4d74-4ae1-b2fa-21d3d0e55834 kind: Event lastTimestamp: "2026-04-16T18:27:46Z" message: 'Created pod: success-200-isvc-df772-predictor-875748d7f-jdp2w' metadata: creationTimestamp: "2026-04-16T18:27:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:27:46Z" name: success-200-isvc-df772-predictor-875748d7f.18a6e9b4d9d6257f namespace: kserve-ci-e2e-test resourceVersion: "20972" uid: 38507a4b-b91a-4bcc-903a-ed61f85e627c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-df772-predictor namespace: kserve-ci-e2e-test resourceVersion: "20952" uid: 4bfaf230-d0ef-45f2-b97c-3b897d22a6f2 kind: Event lastTimestamp: "2026-04-16T18:27:45Z" message: Scaled up replica set success-200-isvc-df772-predictor-875748d7f from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:27:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:27:45Z" name: success-200-isvc-df772-predictor.18a6e9b4d7090dd9 namespace: kserve-ci-e2e-test resourceVersion: "20962" uid: fd4c3f71-31f8-44d7-9f8e-9dff348d7ff9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:28:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-df772-predictor namespace: kserve-ci-e2e-test resourceVersion: "20956" uid: b859dd2f-1f56-4503-803b-7c51cabc44f4 kind: Event lastTimestamp: "2026-04-16T18:28:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:28:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:28:46Z" name: success-200-isvc-df772-predictor.18a6e9b855b9aa6d namespace: kserve-ci-e2e-test resourceVersion: "21739" uid: efd36a55-6b28-4ac9-9797-8dfaeb621796 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:28:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-df772-predictor namespace: kserve-ci-e2e-test resourceVersion: "20956" uid: b859dd2f-1f56-4503-803b-7c51cabc44f4 kind: Event lastTimestamp: "2026-04-16T18:28:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:28:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:28:46Z" name: success-200-isvc-df772-predictor.18a6e9b856183ee2 namespace: kserve-ci-e2e-test resourceVersion: "21740" uid: 4dca4042-576f-4880-b789-b2d201c07030 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:28:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-df772-predictor namespace: kserve-ci-e2e-test resourceVersion: "21188" uid: b859dd2f-1f56-4503-803b-7c51cabc44f4 kind: Event lastTimestamp: "2026-04-16T18:28:31Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:28:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:28:31Z" name: success-200-isvc-df772-predictor.18a6e9bbd51ac6c1 namespace: kserve-ci-e2e-test resourceVersion: "21601" uid: 9db7d65c-a30b-43e7-ae72-818ace299f8e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:28:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-df772-predictor namespace: kserve-ci-e2e-test resourceVersion: "21188" uid: b859dd2f-1f56-4503-803b-7c51cabc44f4 kind: Event lastTimestamp: "2026-04-16T18:28:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:28:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:28:31Z" name: success-200-isvc-df772-predictor.18a6e9bbd58e1f73 namespace: kserve-ci-e2e-test resourceVersion: "21602" uid: ab50662d-8841-4f44-8f1f-24949eafaed3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T18:27:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-df772 namespace: kserve-ci-e2e-test resourceVersion: "20942" uid: 3d04abb9-7218-4f2a-b50e-65fcc596760f kind: Event lastTimestamp: "2026-04-16T18:28:38Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:27:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:28:38Z" name: success-200-isvc-df772.18a6e9b4d72ab41e namespace: kserve-ci-e2e-test resourceVersion: "21674" uid: 17be9b55-8848-475c-b83e-7576b8d073d4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-df772 namespace: kserve-ci-e2e-test resourceVersion: "20942" uid: 3d04abb9-7218-4f2a-b50e-65fcc596760f kind: Event lastTimestamp: "2026-04-16T18:27:45Z" message: 'Failed to update status for InferenceService "success-200-isvc-df772": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-df772": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:27:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:27:45Z" name: success-200-isvc-df772.18a6e9b4d88b5db1 namespace: kserve-ci-e2e-test resourceVersion: "20966" uid: 2c227e3e-f1ee-4fc7-9ae2-03b4885e124e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:27:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-df772 namespace: kserve-ci-e2e-test resourceVersion: "20942" uid: 3d04abb9-7218-4f2a-b50e-65fcc596760f kind: Event lastTimestamp: "2026-04-16T18:27:45Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-df772": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:27:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:27:45Z" name: success-200-isvc-df772.18a6e9b4d88ba750 namespace: kserve-ci-e2e-test resourceVersion: "20970" uid: 948914c1-bdd7-4138-8ef4-ab0340cfc8b3 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-df772 namespace: kserve-ci-e2e-test resourceVersion: "21669" uid: 3d04abb9-7218-4f2a-b50e-65fcc596760f kind: Event lastTimestamp: "2026-04-16T18:28:38Z" message: InferenceService [success-200-isvc-df772] is Ready metadata: creationTimestamp: "2026-04-16T18:28:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:28:38Z" name: success-200-isvc-df772.18a6e9c0f94bc6d4 namespace: kserve-ci-e2e-test resourceVersion: "21671" uid: 847cbb90-9d47-4ac7-8cdf-b7ef56971606 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:28:15.827712Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ed24b-predictor-648cf478d9-kpr96 namespace: kserve-ci-e2e-test resourceVersion: "21393" uid: c8537ca0-09da-4ad3-b1da-78551585c1ee kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-ed24b-predictor-648cf478d9-kpr96 to ip-10-0-128-59.ec2.internal metadata: creationTimestamp: "2026-04-16T18:28:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:28:15Z" name: success-200-isvc-ed24b-predictor-648cf478d9-kpr96.18a6e9bbcb91f10d namespace: kserve-ci-e2e-test resourceVersion: "21398" uid: 55aa8ad5-ab2e-4b47-a798-e3c49ab6be16 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4b7bbd69-nffhf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:16Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ed24b-predictor-648cf478d9-kpr96 namespace: kserve-ci-e2e-test resourceVersion: "21403" uid: c8537ca0-09da-4ad3-b1da-78551585c1ee kind: Event lastTimestamp: "2026-04-16T18:28:16Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:28:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:28:16Z" name: success-200-isvc-ed24b-predictor-648cf478d9-kpr96.18a6e9bbe12d7bac namespace: kserve-ci-e2e-test resourceVersion: "21462" uid: 1a245460-7985-4fe2-a9e3-c75427fd526b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ed24b-predictor-648cf478d9-kpr96 namespace: kserve-ci-e2e-test resourceVersion: "21394" uid: c8537ca0-09da-4ad3-b1da-78551585c1ee kind: Event lastTimestamp: "2026-04-16T18:28:16Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1412" already present on machine metadata: creationTimestamp: "2026-04-16T18:28:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:28:16Z" name: success-200-isvc-ed24b-predictor-648cf478d9-kpr96.18a6e9bbe262021a namespace: kserve-ci-e2e-test resourceVersion: "21464" uid: 3d25cae1-943b-40c1-b03f-0e5bdc462475 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ed24b-predictor-648cf478d9-kpr96 namespace: kserve-ci-e2e-test resourceVersion: "21394" uid: c8537ca0-09da-4ad3-b1da-78551585c1ee kind: Event lastTimestamp: "2026-04-16T18:28:16Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-16T18:28:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:28:16Z" name: success-200-isvc-ed24b-predictor-648cf478d9-kpr96.18a6e9bbe8b2c3d2 namespace: kserve-ci-e2e-test resourceVersion: "21468" uid: 242779ff-05e0-4a8c-bc48-62af5b520bc2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ed24b-predictor-648cf478d9-kpr96 namespace: kserve-ci-e2e-test resourceVersion: "21394" uid: c8537ca0-09da-4ad3-b1da-78551585c1ee kind: Event lastTimestamp: "2026-04-16T18:28:16Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-16T18:28:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:28:16Z" name: success-200-isvc-ed24b-predictor-648cf478d9-kpr96.18a6e9bbe9248d60 namespace: kserve-ci-e2e-test resourceVersion: "21469" uid: 3af229b2-07b2-4ddc-af96-39f1bfb28743 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:28:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ed24b-predictor-648cf478d9-kpr96 namespace: kserve-ci-e2e-test resourceVersion: "21394" uid: c8537ca0-09da-4ad3-b1da-78551585c1ee kind: Event lastTimestamp: "2026-04-16T18:28:58Z" message: 'Readiness probe failed: dial tcp 10.134.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-16T18:28:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:28:58Z" name: success-200-isvc-ed24b-predictor-648cf478d9-kpr96.18a6e9bc16eac7bf namespace: kserve-ci-e2e-test resourceVersion: "21863" uid: 890912ec-a009-4176-a244-ca7378ed9485 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:37:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ed24b-predictor-648cf478d9-kpr96 namespace: kserve-ci-e2e-test resourceVersion: "21394" uid: c8537ca0-09da-4ad3-b1da-78551585c1ee kind: Event lastTimestamp: "2026-04-16T18:37:40Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-16T18:37:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:37:40Z" name: success-200-isvc-ed24b-predictor-648cf478d9-kpr96.18a6ea3f53ba3385 namespace: kserve-ci-e2e-test resourceVersion: "25994" uid: 4cc3c080-ebcb-4688-8760-ac88287578e2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-ed24b-predictor-648cf478d9 namespace: kserve-ci-e2e-test resourceVersion: "21384" uid: e2eaae93-cf17-4fb3-b99a-a7d7a3d04749 kind: Event lastTimestamp: "2026-04-16T18:28:15Z" message: 'Created pod: success-200-isvc-ed24b-predictor-648cf478d9-kpr96' metadata: creationTimestamp: "2026-04-16T18:28:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:28:15Z" name: success-200-isvc-ed24b-predictor-648cf478d9.18a6e9bbcb26e4ec namespace: kserve-ci-e2e-test resourceVersion: "21395" uid: c24ab2c8-c144-44e1-9cb2-64d5c152bb43 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-ed24b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21377" uid: ccb73301-98c4-4a8c-a845-b612f88f9172 kind: Event lastTimestamp: "2026-04-16T18:28:15Z" message: Scaled up replica set success-200-isvc-ed24b-predictor-648cf478d9 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:28:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:28:15Z" name: success-200-isvc-ed24b-predictor.18a6e9bbc8fa8595 namespace: kserve-ci-e2e-test resourceVersion: "21387" uid: 749ddf86-f4fe-4658-a4c5-90c24d2f91e8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:28:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ed24b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21383" uid: 7d5be51a-ee75-42ce-bad9-08b4a71365e2 kind: Event lastTimestamp: "2026-04-16T18:29:00Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:28:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:29:00Z" name: success-200-isvc-ed24b-predictor.18a6e9bf479c5b96 namespace: kserve-ci-e2e-test resourceVersion: "21883" uid: d85f7f0b-b64c-4bd3-b0c4-4c91a4ac34be reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-16T18:28:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ed24b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21383" uid: 7d5be51a-ee75-42ce-bad9-08b4a71365e2 kind: Event lastTimestamp: "2026-04-16T18:29:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:28:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:29:00Z" name: success-200-isvc-ed24b-predictor.18a6e9bf480c1776 namespace: kserve-ci-e2e-test resourceVersion: "21884" uid: f5564feb-be85-42f8-b497-e027ca8e36a4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:29:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ed24b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21595" uid: 7d5be51a-ee75-42ce-bad9-08b4a71365e2 kind: Event lastTimestamp: "2026-04-16T18:29:30Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:29:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:29:30Z" name: success-200-isvc-ed24b-predictor.18a6e9c9c4d59994 namespace: kserve-ci-e2e-test resourceVersion: "22168" uid: 8da1d999-8537-461e-bc3e-d4fcd9e095a5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-16T18:29:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ed24b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21595" uid: 7d5be51a-ee75-42ce-bad9-08b4a71365e2 kind: Event lastTimestamp: "2026-04-16T18:29:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:29:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:29:30Z" name: success-200-isvc-ed24b-predictor.18a6e9c9c52aec69 namespace: kserve-ci-e2e-test resourceVersion: "22169" uid: 21fb3fa2-397d-4136-94b9-a60b47a33259 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T18:28:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ed24b namespace: kserve-ci-e2e-test resourceVersion: "21363" uid: 7867f808-178e-4069-b708-1da53dfe493d kind: Event lastTimestamp: "2026-04-16T18:29:08Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-16T18:28:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:29:08Z" name: success-200-isvc-ed24b.18a6e9bbc8ec92cf namespace: kserve-ci-e2e-test resourceVersion: "21964" uid: db58a37e-00be-45b7-bda4-1d076a2b62b9 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ed24b namespace: kserve-ci-e2e-test resourceVersion: "21363" uid: 7867f808-178e-4069-b708-1da53dfe493d kind: Event lastTimestamp: "2026-04-16T18:28:15Z" message: 'Failed to update status for InferenceService "success-200-isvc-ed24b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-ed24b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:28:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:28:15Z" name: success-200-isvc-ed24b.18a6e9bbc9c6dbc0 namespace: kserve-ci-e2e-test resourceVersion: "21389" uid: 287676f5-65c1-463c-b2ee-b96f2936f43d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:28:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ed24b namespace: kserve-ci-e2e-test resourceVersion: "21363" uid: 7867f808-178e-4069-b708-1da53dfe493d kind: Event lastTimestamp: "2026-04-16T18:28:15Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-ed24b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-16T18:28:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:28:15Z" name: success-200-isvc-ed24b.18a6e9bbc9c7377c namespace: kserve-ci-e2e-test resourceVersion: "21392" uid: 2a553ce1-4a64-4892-9974-942a2c949620 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:29:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ed24b namespace: kserve-ci-e2e-test resourceVersion: "21962" uid: 7867f808-178e-4069-b708-1da53dfe493d kind: Event lastTimestamp: "2026-04-16T18:29:08Z" message: InferenceService [success-200-isvc-ed24b] is Ready metadata: creationTimestamp: "2026-04-16T18:29:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:29:08Z" name: success-200-isvc-ed24b.18a6e9c7fb2c114c namespace: kserve-ci-e2e-test resourceVersion: "21963" uid: 565b6da1-e4a6-480e-b5c6-29c866a1de7b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:18:10.821071Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-04c3d-b98467bdb-ssm57 namespace: kserve-ci-e2e-test resourceVersion: "16495" uid: e559e62e-8fb8-4de9-9ecf-54ca36d53802 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-04c3d-b98467bdb-ssm57 to ip-10-0-128-59.ec2.internal metadata: creationTimestamp: "2026-04-16T18:18:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:18:10Z" name: switch-graph-04c3d-b98467bdb-ssm57.18a6e92eee5cfffb namespace: kserve-ci-e2e-test resourceVersion: "16500" uid: c3e473c4-318b-408c-b329-ae4cee2e5a7c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4b7bbd69-nffhf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:11Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-04c3d-b98467bdb-ssm57 namespace: kserve-ci-e2e-test resourceVersion: "16504" uid: e559e62e-8fb8-4de9-9ecf-54ca36d53802 kind: Event lastTimestamp: "2026-04-16T18:18:11Z" message: Add eth0 [10.134.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:18:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:18:11Z" name: switch-graph-04c3d-b98467bdb-ssm57.18a6e92f076d4d87 namespace: kserve-ci-e2e-test resourceVersion: "16514" uid: 92ded494-728d-443e-9e75-3e36525af676 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-04c3d} kind: Pod name: switch-graph-04c3d-b98467bdb-ssm57 namespace: kserve-ci-e2e-test resourceVersion: "16498" uid: e559e62e-8fb8-4de9-9ecf-54ca36d53802 kind: Event lastTimestamp: "2026-04-16T18:18:11Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:6060da2148ba78c06d6de96222351f1e5f7d3ada4027ac02142aa0e1928e4f29" already present on machine metadata: creationTimestamp: "2026-04-16T18:18:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:18:11Z" name: switch-graph-04c3d-b98467bdb-ssm57.18a6e92f08a330f3 namespace: kserve-ci-e2e-test resourceVersion: "16516" uid: a8b69271-14b7-4b07-a5ec-4903d6d5d26a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-04c3d} kind: Pod name: switch-graph-04c3d-b98467bdb-ssm57 namespace: kserve-ci-e2e-test resourceVersion: "16498" uid: e559e62e-8fb8-4de9-9ecf-54ca36d53802 kind: Event lastTimestamp: "2026-04-16T18:18:11Z" message: 'Created container: switch-graph-04c3d' metadata: creationTimestamp: "2026-04-16T18:18:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:18:11Z" name: switch-graph-04c3d-b98467bdb-ssm57.18a6e92f0bc4977d namespace: kserve-ci-e2e-test resourceVersion: "16517" uid: 503135eb-9dc6-4f1d-a800-6903e05c0776 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-04c3d} kind: Pod name: switch-graph-04c3d-b98467bdb-ssm57 namespace: kserve-ci-e2e-test resourceVersion: "16498" uid: e559e62e-8fb8-4de9-9ecf-54ca36d53802 kind: Event lastTimestamp: "2026-04-16T18:18:11Z" message: Started container switch-graph-04c3d metadata: creationTimestamp: "2026-04-16T18:18:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:18:11Z" name: switch-graph-04c3d-b98467bdb-ssm57.18a6e92f0c408670 namespace: kserve-ci-e2e-test resourceVersion: "16518" uid: a6e9b032-190b-4cfe-826c-c11bfc6c9ed7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:26:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-04c3d} kind: Pod name: switch-graph-04c3d-b98467bdb-ssm57 namespace: kserve-ci-e2e-test resourceVersion: "16498" uid: e559e62e-8fb8-4de9-9ecf-54ca36d53802 kind: Event lastTimestamp: "2026-04-16T18:26:25Z" message: Stopping container switch-graph-04c3d metadata: creationTimestamp: "2026-04-16T18:26:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:26:25Z" name: switch-graph-04c3d-b98467bdb-ssm57.18a6e9a21e088f94 namespace: kserve-ci-e2e-test resourceVersion: "19990" uid: 72c9f4e1-0d99-4e27-a0f7-6c7cb961f6ff reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:26:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-04c3d} kind: Pod name: switch-graph-04c3d-b98467bdb-ssm57 namespace: kserve-ci-e2e-test resourceVersion: "16498" uid: e559e62e-8fb8-4de9-9ecf-54ca36d53802 kind: Event lastTimestamp: "2026-04-16T18:26:52Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T18:26:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:26:52Z" name: switch-graph-04c3d-b98467bdb-ssm57.18a6e9a28a8bf03a namespace: kserve-ci-e2e-test resourceVersion: "20300" uid: 142bbdb3-e554-44e6-a4bb-d62696a7166b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-04c3d-b98467bdb namespace: kserve-ci-e2e-test resourceVersion: "16492" uid: b13c5cd5-b249-4ec6-8b2a-42d912608303 kind: Event lastTimestamp: "2026-04-16T18:18:10Z" message: 'Created pod: switch-graph-04c3d-b98467bdb-ssm57' metadata: creationTimestamp: "2026-04-16T18:18:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:18:10Z" name: switch-graph-04c3d-b98467bdb.18a6e92eedc39a63 namespace: kserve-ci-e2e-test resourceVersion: "16497" uid: a027b617-91b3-40fb-bd2d-f1ee6464ff02 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-04c3d namespace: kserve-ci-e2e-test resourceVersion: "16490" uid: d7b577f5-2a81-4470-9350-af1048e24a05 kind: Event lastTimestamp: "2026-04-16T18:18:10Z" message: Scaled up replica set switch-graph-04c3d-b98467bdb from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:18:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:18:10Z" name: switch-graph-04c3d.18a6e92eec8b8f31 namespace: kserve-ci-e2e-test resourceVersion: "16493" uid: cf01539d-dbf3-41f2-82c4-5355673f86ff reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:10Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-04c3d namespace: kserve-ci-e2e-test resourceVersion: "16506" uid: 869e2e1c-c005-491b-9459-624de29a8e15 kind: Event lastTimestamp: "2026-04-16T18:18:10Z" message: InferenceGraph [switch-graph-04c3d] is Ready metadata: creationTimestamp: "2026-04-16T18:18:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:18:10Z" name: switch-graph-04c3d.18a6e92ef3128320 namespace: kserve-ci-e2e-test resourceVersion: "16507" uid: d7214fef-8991-4651-8a4d-fe09ef03f46a reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-04c3d namespace: kserve-ci-e2e-test resourceVersion: "16491" uid: 1d728cb7-2e4e-43b7-ba80-9eb72e8f9430 kind: Event lastTimestamp: "2026-04-16T18:18:25Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:18:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:18:25Z" name: switch-graph-04c3d.18a6e9326b22f4e4 namespace: kserve-ci-e2e-test resourceVersion: "16660" uid: bc309d8e-8057-4642-b081-887803d89064 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-04c3d namespace: kserve-ci-e2e-test resourceVersion: "16491" uid: 1d728cb7-2e4e-43b7-ba80-9eb72e8f9430 kind: Event lastTimestamp: "2026-04-16T18:18:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:18:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:18:25Z" name: switch-graph-04c3d.18a6e9326b91075c namespace: kserve-ci-e2e-test resourceVersion: "16662" uid: 143d3fa1-a39e-45fb-bcfd-bf7e299d1e90 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-04c3d namespace: kserve-ci-e2e-test resourceVersion: "16661" uid: 1d728cb7-2e4e-43b7-ba80-9eb72e8f9430 kind: Event lastTimestamp: "2026-04-16T18:18:40Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:18:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:18:40Z" name: switch-graph-04c3d.18a6e935eaa8f37d namespace: kserve-ci-e2e-test resourceVersion: "16798" uid: dd3d1961-da88-4276-9be8-1d264f1b43b0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:18:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-04c3d namespace: kserve-ci-e2e-test resourceVersion: "16661" uid: 1d728cb7-2e4e-43b7-ba80-9eb72e8f9430 kind: Event lastTimestamp: "2026-04-16T18:18:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:18:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:18:40Z" name: switch-graph-04c3d.18a6e935eb13a223 namespace: kserve-ci-e2e-test resourceVersion: "16800" uid: ece28cd4-077c-4f29-b8bd-27df04506914 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:16:50.285022Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-390f9-67fbf8748c-gm76t namespace: kserve-ci-e2e-test resourceVersion: "15441" uid: 37b7bbd3-8d23-42c5-b721-78044f2e91a8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-390f9-67fbf8748c-gm76t to ip-10-0-128-59.ec2.internal metadata: creationTimestamp: "2026-04-16T18:16:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:16:50Z" name: switch-graph-390f9-67fbf8748c-gm76t.18a6e91c2e0a72c6 namespace: kserve-ci-e2e-test resourceVersion: "15447" uid: 6b592672-944f-4066-b3ad-c00db7c92b29 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4b7bbd69-nffhf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:16:50Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-390f9-67fbf8748c-gm76t namespace: kserve-ci-e2e-test resourceVersion: "15444" uid: 37b7bbd3-8d23-42c5-b721-78044f2e91a8 kind: Event lastTimestamp: "2026-04-16T18:16:50Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-390f9-serving-cert" not found' metadata: creationTimestamp: "2026-04-16T18:16:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:16:50Z" name: switch-graph-390f9-67fbf8748c-gm76t.18a6e91c3cdbc677 namespace: kserve-ci-e2e-test resourceVersion: "15459" uid: 704123b9-c17b-4663-99b2-f515d5dca263 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:16:51Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-390f9-67fbf8748c-gm76t namespace: kserve-ci-e2e-test resourceVersion: "15450" uid: 37b7bbd3-8d23-42c5-b721-78044f2e91a8 kind: Event lastTimestamp: "2026-04-16T18:16:51Z" message: Add eth0 [10.134.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:16:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:16:51Z" name: switch-graph-390f9-67fbf8748c-gm76t.18a6e91c6ada20d4 namespace: kserve-ci-e2e-test resourceVersion: "15466" uid: f210b6d1-00d3-4a7e-8fb3-59c3b46e6932 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:16:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-390f9} kind: Pod name: switch-graph-390f9-67fbf8748c-gm76t namespace: kserve-ci-e2e-test resourceVersion: "15444" uid: 37b7bbd3-8d23-42c5-b721-78044f2e91a8 kind: Event lastTimestamp: "2026-04-16T18:16:51Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:6060da2148ba78c06d6de96222351f1e5f7d3ada4027ac02142aa0e1928e4f29" metadata: creationTimestamp: "2026-04-16T18:16:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:16:51Z" name: switch-graph-390f9-67fbf8748c-gm76t.18a6e91c6c0dec3c namespace: kserve-ci-e2e-test resourceVersion: "15468" uid: 4ce9bafe-0e6b-4a11-b7a0-4ac185fe0428 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:16:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-390f9} kind: Pod name: switch-graph-390f9-67fbf8748c-gm76t namespace: kserve-ci-e2e-test resourceVersion: "15444" uid: 37b7bbd3-8d23-42c5-b721-78044f2e91a8 kind: Event lastTimestamp: "2026-04-16T18:16:53Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:6060da2148ba78c06d6de96222351f1e5f7d3ada4027ac02142aa0e1928e4f29" in 2.411s (2.411s including waiting). Image size: 216232023 bytes.' metadata: creationTimestamp: "2026-04-16T18:16:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:16:53Z" name: switch-graph-390f9-67fbf8748c-gm76t.18a6e91cfbd094a7 namespace: kserve-ci-e2e-test resourceVersion: "15485" uid: ff0a0598-2b5d-473b-8e0a-5cef9ffbbbd0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:16:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-390f9} kind: Pod name: switch-graph-390f9-67fbf8748c-gm76t namespace: kserve-ci-e2e-test resourceVersion: "15444" uid: 37b7bbd3-8d23-42c5-b721-78044f2e91a8 kind: Event lastTimestamp: "2026-04-16T18:16:53Z" message: 'Created container: switch-graph-390f9' metadata: creationTimestamp: "2026-04-16T18:16:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:16:53Z" name: switch-graph-390f9-67fbf8748c-gm76t.18a6e91cff02058b namespace: kserve-ci-e2e-test resourceVersion: "15487" uid: 2dc23761-3b25-440a-936a-512c39188930 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:16:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-390f9} kind: Pod name: switch-graph-390f9-67fbf8748c-gm76t namespace: kserve-ci-e2e-test resourceVersion: "15444" uid: 37b7bbd3-8d23-42c5-b721-78044f2e91a8 kind: Event lastTimestamp: "2026-04-16T18:16:53Z" message: Started container switch-graph-390f9 metadata: creationTimestamp: "2026-04-16T18:16:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:16:53Z" name: switch-graph-390f9-67fbf8748c-gm76t.18a6e91cff7f8683 namespace: kserve-ci-e2e-test resourceVersion: "15488" uid: 102a3cfc-92d8-43b5-a600-df51ede4f401 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:17:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-390f9} kind: Pod name: switch-graph-390f9-67fbf8748c-gm76t namespace: kserve-ci-e2e-test resourceVersion: "15444" uid: 37b7bbd3-8d23-42c5-b721-78044f2e91a8 kind: Event lastTimestamp: "2026-04-16T18:17:00Z" message: Stopping container switch-graph-390f9 metadata: creationTimestamp: "2026-04-16T18:17:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:17:00Z" name: switch-graph-390f9-67fbf8748c-gm76t.18a6e91e8dcd79ba namespace: kserve-ci-e2e-test resourceVersion: "15576" uid: 3272d799-e852-4105-896b-485954992e1a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-16T18:17:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-390f9} kind: Pod name: switch-graph-390f9-67fbf8748c-gm76t namespace: kserve-ci-e2e-test resourceVersion: "15444" uid: 37b7bbd3-8d23-42c5-b721-78044f2e91a8 kind: Event lastTimestamp: "2026-04-16T18:17:30Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T18:17:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:17:30Z" name: switch-graph-390f9-67fbf8748c-gm76t.18a6e91fa24afb8e namespace: kserve-ci-e2e-test resourceVersion: "15990" uid: 83b83bda-929a-47a2-bdcb-22d583b4aa36 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:16:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-390f9-67fbf8748c namespace: kserve-ci-e2e-test resourceVersion: "15438" uid: b86c9708-7f39-4ee9-a213-bd84bd1e79c8 kind: Event lastTimestamp: "2026-04-16T18:16:50Z" message: 'Created pod: switch-graph-390f9-67fbf8748c-gm76t' metadata: creationTimestamp: "2026-04-16T18:16:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:16:50Z" name: switch-graph-390f9-67fbf8748c.18a6e91c2d439f7c namespace: kserve-ci-e2e-test resourceVersion: "15443" uid: 5f60a7fc-34ea-43eb-909b-23a0c02dcb04 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:16:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-390f9 namespace: kserve-ci-e2e-test resourceVersion: "15436" uid: efaf88cf-bd5d-4f16-8826-b1c10a512813 kind: Event lastTimestamp: "2026-04-16T18:16:50Z" message: Scaled up replica set switch-graph-390f9-67fbf8748c from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:16:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:16:50Z" name: switch-graph-390f9.18a6e91c2c277acb namespace: kserve-ci-e2e-test resourceVersion: "15439" uid: 907a8a1d-9739-48de-829e-bb00e469d4f4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:16:50Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-390f9 namespace: kserve-ci-e2e-test resourceVersion: "15452" uid: d0c1f453-1a8b-4a4a-b2b7-3fedb92ff356 kind: Event lastTimestamp: "2026-04-16T18:16:50Z" message: InferenceGraph [switch-graph-390f9] is Ready metadata: creationTimestamp: "2026-04-16T18:16:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:16:50Z" name: switch-graph-390f9.18a6e91c3277ad9f namespace: kserve-ci-e2e-test resourceVersion: "15455" uid: e1b685c8-ebd5-46c1-9f82-1d4033600a0e reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-16T18:38:50.802574Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-89f53-d7bd89598-qk2zv namespace: kserve-ci-e2e-test resourceVersion: "26856" uid: 148cab96-c158-4ff5-9f75-51c5a9d73396 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-89f53-d7bd89598-qk2zv to ip-10-0-128-59.ec2.internal metadata: creationTimestamp: "2026-04-16T18:38:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-16T18:38:50Z" name: switch-graph-89f53-d7bd89598-qk2zv.18a6ea4fa3053cca namespace: kserve-ci-e2e-test resourceVersion: "26861" uid: 82154699-48f0-4806-8915-5e9c61d8e9d4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b4b7bbd69-nffhf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:50Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-89f53-d7bd89598-qk2zv namespace: kserve-ci-e2e-test resourceVersion: "26858" uid: 148cab96-c158-4ff5-9f75-51c5a9d73396 kind: Event lastTimestamp: "2026-04-16T18:38:50Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-89f53-serving-cert" not found' metadata: creationTimestamp: "2026-04-16T18:38:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:38:50Z" name: switch-graph-89f53-d7bd89598-qk2zv.18a6ea4faba416cc namespace: kserve-ci-e2e-test resourceVersion: "26873" uid: aa89b93f-2ec1-4c6e-a6a3-80832165f478 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:51Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-89f53-d7bd89598-qk2zv namespace: kserve-ci-e2e-test resourceVersion: "26865" uid: 148cab96-c158-4ff5-9f75-51c5a9d73396 kind: Event lastTimestamp: "2026-04-16T18:38:51Z" message: Add eth0 [10.134.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-16T18:38:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-16T18:38:51Z" name: switch-graph-89f53-d7bd89598-qk2zv.18a6ea4fe1945388 namespace: kserve-ci-e2e-test resourceVersion: "26881" uid: 68c7ca0d-9f3b-4f2d-8c36-679fcde89f61 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-89f53} kind: Pod name: switch-graph-89f53-d7bd89598-qk2zv namespace: kserve-ci-e2e-test resourceVersion: "26858" uid: 148cab96-c158-4ff5-9f75-51c5a9d73396 kind: Event lastTimestamp: "2026-04-16T18:38:51Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:6060da2148ba78c06d6de96222351f1e5f7d3ada4027ac02142aa0e1928e4f29" already present on machine metadata: creationTimestamp: "2026-04-16T18:38:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:38:51Z" name: switch-graph-89f53-d7bd89598-qk2zv.18a6ea4fe2bdb266 namespace: kserve-ci-e2e-test resourceVersion: "26883" uid: 3c90f769-e491-45ca-bb1c-ccbe7accd9f1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-89f53} kind: Pod name: switch-graph-89f53-d7bd89598-qk2zv namespace: kserve-ci-e2e-test resourceVersion: "26858" uid: 148cab96-c158-4ff5-9f75-51c5a9d73396 kind: Event lastTimestamp: "2026-04-16T18:38:51Z" message: 'Created container: switch-graph-89f53' metadata: creationTimestamp: "2026-04-16T18:38:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:38:51Z" name: switch-graph-89f53-d7bd89598-qk2zv.18a6ea4fe61aa7fb namespace: kserve-ci-e2e-test resourceVersion: "26885" uid: fb9dd7d7-4466-4986-8f33-339d0263ca27 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-89f53} kind: Pod name: switch-graph-89f53-d7bd89598-qk2zv namespace: kserve-ci-e2e-test resourceVersion: "26858" uid: 148cab96-c158-4ff5-9f75-51c5a9d73396 kind: Event lastTimestamp: "2026-04-16T18:38:51Z" message: Started container switch-graph-89f53 metadata: creationTimestamp: "2026-04-16T18:38:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:38:51Z" name: switch-graph-89f53-d7bd89598-qk2zv.18a6ea4fe6da4aaa namespace: kserve-ci-e2e-test resourceVersion: "26886" uid: f3cc2689-d7d5-4ffc-87dc-a4b7a6160a39 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:55:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-89f53} kind: Pod name: switch-graph-89f53-d7bd89598-qk2zv namespace: kserve-ci-e2e-test resourceVersion: "26858" uid: 148cab96-c158-4ff5-9f75-51c5a9d73396 kind: Event lastTimestamp: "2026-04-16T18:55:10Z" message: Stopping container switch-graph-89f53 metadata: creationTimestamp: "2026-04-16T18:55:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:55:10Z" name: switch-graph-89f53-d7bd89598-qk2zv.18a6eb33ae026960 namespace: kserve-ci-e2e-test resourceVersion: "33764" uid: e54da358-b546-4206-9f6b-6fd6f3f07dd8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-16T18:55:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-89f53} kind: Pod name: switch-graph-89f53-d7bd89598-qk2zv namespace: kserve-ci-e2e-test resourceVersion: "26858" uid: 148cab96-c158-4ff5-9f75-51c5a9d73396 kind: Event lastTimestamp: "2026-04-16T18:55:33Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-16T18:55:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-16T18:55:33Z" name: switch-graph-89f53-d7bd89598-qk2zv.18a6eb34897569cb namespace: kserve-ci-e2e-test resourceVersion: "34002" uid: 6a739783-8446-494d-bbcb-a43c6f3bca52 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-59.ec2.internal source: component: kubelet host: ip-10-0-128-59.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-89f53-d7bd89598 namespace: kserve-ci-e2e-test resourceVersion: "26853" uid: b05adffe-6226-48f7-a636-0826a1c33fcb kind: Event lastTimestamp: "2026-04-16T18:38:50Z" message: 'Created pod: switch-graph-89f53-d7bd89598-qk2zv' metadata: creationTimestamp: "2026-04-16T18:38:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:38:50Z" name: switch-graph-89f53-d7bd89598.18a6ea4fa2a104ac namespace: kserve-ci-e2e-test resourceVersion: "26857" uid: b2e4173d-f4a7-436a-9407-32230ca7397f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-89f53 namespace: kserve-ci-e2e-test resourceVersion: "26851" uid: 682cdd9b-acd1-4384-afa7-eeb941d37979 kind: Event lastTimestamp: "2026-04-16T18:38:50Z" message: Scaled up replica set switch-graph-89f53-d7bd89598 from 0 to 1 metadata: creationTimestamp: "2026-04-16T18:38:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:38:50Z" name: switch-graph-89f53.18a6ea4fa20c838e namespace: kserve-ci-e2e-test resourceVersion: "26854" uid: 7c63b6c6-2a68-40e6-94fb-cfa121f8ecc9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:38:50Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-89f53 namespace: kserve-ci-e2e-test resourceVersion: "26867" uid: 29c48c9e-30a4-45e5-9bb7-3b739cea40ec kind: Event lastTimestamp: "2026-04-16T18:38:50Z" message: InferenceGraph [switch-graph-89f53] is Ready metadata: creationTimestamp: "2026-04-16T18:38:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-16T18:38:50Z" name: switch-graph-89f53.18a6ea4fa71efea0 namespace: kserve-ci-e2e-test resourceVersion: "26869" uid: b92a38ab-1390-4d00-ba97-29022599935c reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-89f53 namespace: kserve-ci-e2e-test resourceVersion: "26852" uid: 0c5a8f3a-f3b3-401a-a4ee-ad2e56d90f2e kind: Event lastTimestamp: "2026-04-16T18:39:05Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:39:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:39:05Z" name: switch-graph-89f53.18a6ea532099a1ef namespace: kserve-ci-e2e-test resourceVersion: "27006" uid: 86e53fda-a1d7-48f6-aa7e-dcf758784461 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-89f53 namespace: kserve-ci-e2e-test resourceVersion: "26852" uid: 0c5a8f3a-f3b3-401a-a4ee-ad2e56d90f2e kind: Event lastTimestamp: "2026-04-16T18:39:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-16T18:39:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:39:05Z" name: switch-graph-89f53.18a6ea532106e248 namespace: kserve-ci-e2e-test resourceVersion: "27008" uid: c6f58fb1-0aa6-4ce8-9d6e-39340ca02690 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-89f53 namespace: kserve-ci-e2e-test resourceVersion: "27007" uid: 0c5a8f3a-f3b3-401a-a4ee-ad2e56d90f2e kind: Event lastTimestamp: "2026-04-16T18:39:20Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:39:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:39:20Z" name: switch-graph-89f53.18a6ea569fb9d444 namespace: kserve-ci-e2e-test resourceVersion: "27121" uid: 0c808312-0e17-4416-a9a5-c29a62a97316 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-16T18:39:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-89f53 namespace: kserve-ci-e2e-test resourceVersion: "27007" uid: 0c5a8f3a-f3b3-401a-a4ee-ad2e56d90f2e kind: Event lastTimestamp: "2026-04-16T18:39:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-16T18:39:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-16T18:39:20Z" name: switch-graph-89f53.18a6ea56a00d2fa0 namespace: kserve-ci-e2e-test resourceVersion: "27123" uid: 2e817eef-f235-414e-8e79-60964d0083fc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning kind: EventList metadata: resourceVersion: "34002"