--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-20T10:20:46.544221Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-242a5-75b5874dcc-scmgx namespace: kserve-ci-e2e-test resourceVersion: "21058" uid: fccc48c2-8b8c-4513-8748-4a3d803325cf kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-242a5-75b5874dcc-scmgx to ip-10-0-138-143.ec2.internal metadata: creationTimestamp: "2026-07-20T10:20:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-20T10:20:46Z" name: ensemble-graph-242a5-75b5874dcc-scmgx.18c3f84396bbafd6 namespace: kserve-ci-e2e-test resourceVersion: "21063" uid: 1601e022-3bc0-47cf-9981-60aaaa0c4899 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5dc669d9-69hkz source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:46Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-242a5-75b5874dcc-scmgx namespace: kserve-ci-e2e-test resourceVersion: "21059" uid: fccc48c2-8b8c-4513-8748-4a3d803325cf kind: Event lastTimestamp: "2026-07-20T10:20:46Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-242a5-serving-cert" not found' metadata: creationTimestamp: "2026-07-20T10:20:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:20:46Z" name: ensemble-graph-242a5-75b5874dcc-scmgx.18c3f843a86eb68c namespace: kserve-ci-e2e-test resourceVersion: "21080" uid: 5e8879fa-db24-42db-b5f1-464624c2246f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:47Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-242a5-75b5874dcc-scmgx namespace: kserve-ci-e2e-test resourceVersion: "21067" uid: fccc48c2-8b8c-4513-8748-4a3d803325cf kind: Event lastTimestamp: "2026-07-20T10:20:47Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-20T10:20:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-20T10:20:47Z" name: ensemble-graph-242a5-75b5874dcc-scmgx.18c3f843d3a9c245 namespace: kserve-ci-e2e-test resourceVersion: "21091" uid: bd6e3522-8d60-4407-a157-a1f69304296b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-242a5} kind: Pod name: ensemble-graph-242a5-75b5874dcc-scmgx namespace: kserve-ci-e2e-test resourceVersion: "21059" uid: fccc48c2-8b8c-4513-8748-4a3d803325cf kind: Event lastTimestamp: "2026-07-20T10:20:47Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:ee73d7a22df8b79aa8a5e917cb435b1d2b100d4582439c437bdc01a1d2422290" already present on machine metadata: creationTimestamp: "2026-07-20T10:20:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:20:47Z" name: ensemble-graph-242a5-75b5874dcc-scmgx.18c3f843d505d823 namespace: kserve-ci-e2e-test resourceVersion: "21093" uid: 4e794cd1-393d-4d60-9724-dbb8d4bd0962 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-242a5} kind: Pod name: ensemble-graph-242a5-75b5874dcc-scmgx namespace: kserve-ci-e2e-test resourceVersion: "21059" uid: fccc48c2-8b8c-4513-8748-4a3d803325cf kind: Event lastTimestamp: "2026-07-20T10:20:47Z" message: 'Created container: ensemble-graph-242a5' metadata: creationTimestamp: "2026-07-20T10:20:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:20:47Z" name: ensemble-graph-242a5-75b5874dcc-scmgx.18c3f843d880c429 namespace: kserve-ci-e2e-test resourceVersion: "21096" uid: dda32ff4-dc40-455c-ae30-8430e18a67c7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-242a5} kind: Pod name: ensemble-graph-242a5-75b5874dcc-scmgx namespace: kserve-ci-e2e-test resourceVersion: "21059" uid: fccc48c2-8b8c-4513-8748-4a3d803325cf kind: Event lastTimestamp: "2026-07-20T10:20:47Z" message: Started container ensemble-graph-242a5 metadata: creationTimestamp: "2026-07-20T10:20:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:20:47Z" name: ensemble-graph-242a5-75b5874dcc-scmgx.18c3f843d8f5b771 namespace: kserve-ci-e2e-test resourceVersion: "21097" uid: 7c1e889f-d54c-4124-ba9f-6fb3087b2109 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-242a5} kind: Pod name: ensemble-graph-242a5-75b5874dcc-scmgx namespace: kserve-ci-e2e-test resourceVersion: "21059" uid: fccc48c2-8b8c-4513-8748-4a3d803325cf kind: Event lastTimestamp: "2026-07-20T10:20:56Z" message: Stopping container ensemble-graph-242a5 metadata: creationTimestamp: "2026-07-20T10:20:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:20:56Z" name: ensemble-graph-242a5-75b5874dcc-scmgx.18c3f845ef39b453 namespace: kserve-ci-e2e-test resourceVersion: "21189" uid: 5d9a53d0-0024-4cbf-973e-5bca4def15fc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-20T10:20:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-242a5} kind: Pod name: ensemble-graph-242a5-75b5874dcc-scmgx namespace: kserve-ci-e2e-test resourceVersion: "21059" uid: fccc48c2-8b8c-4513-8748-4a3d803325cf kind: Event lastTimestamp: "2026-07-20T10:21:24Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-20T10:20:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:21:24Z" name: ensemble-graph-242a5-75b5874dcc-scmgx.18c3f8467e3d053b namespace: kserve-ci-e2e-test resourceVersion: "21548" uid: 0295ef9b-6c4a-42c7-8198-4217650ee02f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:46Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-242a5-75b5874dcc namespace: kserve-ci-e2e-test resourceVersion: "21055" uid: 264f58fa-0881-445f-b571-acfaf4b5ca4a kind: Event lastTimestamp: "2026-07-20T10:20:46Z" message: 'Created pod: ensemble-graph-242a5-75b5874dcc-scmgx' metadata: creationTimestamp: "2026-07-20T10:20:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:20:46Z" name: ensemble-graph-242a5-75b5874dcc.18c3f8439661ff0c namespace: kserve-ci-e2e-test resourceVersion: "21060" uid: 36756030-e462-45cc-83a2-36d71868de6a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:46Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-242a5 namespace: kserve-ci-e2e-test resourceVersion: "21053" uid: 824f1817-8f05-4f26-9a77-6301baf2df2e kind: Event lastTimestamp: "2026-07-20T10:20:46Z" message: Scaled up replica set ensemble-graph-242a5-75b5874dcc from 0 to 1 metadata: creationTimestamp: "2026-07-20T10:20:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:20:46Z" name: ensemble-graph-242a5.18c3f843958741e1 namespace: kserve-ci-e2e-test resourceVersion: "21056" uid: c0ff0e55-a83c-4124-bfcc-5f396060a180 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:46Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-242a5 namespace: kserve-ci-e2e-test resourceVersion: "21070" uid: 037968a3-fc18-48ce-ab2d-3613fde0ed14 kind: Event lastTimestamp: "2026-07-20T10:20:46Z" message: InferenceGraph [ensemble-graph-242a5] is Ready metadata: creationTimestamp: "2026-07-20T10:20:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:20:46Z" name: ensemble-graph-242a5.18c3f8439ac04072 namespace: kserve-ci-e2e-test resourceVersion: "21073" uid: 9ea87ace-7174-4edc-8333-765d06050b39 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-20T10:21:56.882771Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-53302-75ff947f68-lkcrm namespace: kserve-ci-e2e-test resourceVersion: "21995" uid: 683dbf0e-ba1a-45e7-83a1-28813d7c4bdc kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-53302-75ff947f68-lkcrm to ip-10-0-138-143.ec2.internal metadata: creationTimestamp: "2026-07-20T10:21:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-20T10:21:56Z" name: ensemble-graph-53302-75ff947f68-lkcrm.18c3f853f73cca91 namespace: kserve-ci-e2e-test resourceVersion: "22002" uid: 4cfb1955-d5fa-47d9-89d1-c02f96d5c51e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5dc669d9-69hkz source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:57Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-53302-75ff947f68-lkcrm namespace: kserve-ci-e2e-test resourceVersion: "21997" uid: 683dbf0e-ba1a-45e7-83a1-28813d7c4bdc kind: Event lastTimestamp: "2026-07-20T10:21:57Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-53302-serving-cert" not found' metadata: creationTimestamp: "2026-07-20T10:21:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:21:57Z" name: ensemble-graph-53302-75ff947f68-lkcrm.18c3f854030df9b7 namespace: kserve-ci-e2e-test resourceVersion: "22020" uid: 4b391ba6-c478-4100-bf0c-09ea6388cb5d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:57Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-53302-75ff947f68-lkcrm namespace: kserve-ci-e2e-test resourceVersion: "22005" uid: 683dbf0e-ba1a-45e7-83a1-28813d7c4bdc kind: Event lastTimestamp: "2026-07-20T10:21:57Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-20T10:21:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-20T10:21:57Z" name: ensemble-graph-53302-75ff947f68-lkcrm.18c3f8543452c509 namespace: kserve-ci-e2e-test resourceVersion: "22041" uid: 963b92f1-c517-4065-8f75-296d8ea5a5bc reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-53302} kind: Pod name: ensemble-graph-53302-75ff947f68-lkcrm namespace: kserve-ci-e2e-test resourceVersion: "21997" uid: 683dbf0e-ba1a-45e7-83a1-28813d7c4bdc kind: Event lastTimestamp: "2026-07-20T10:21:57Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:ee73d7a22df8b79aa8a5e917cb435b1d2b100d4582439c437bdc01a1d2422290" already present on machine metadata: creationTimestamp: "2026-07-20T10:21:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:21:57Z" name: ensemble-graph-53302-75ff947f68-lkcrm.18c3f85435d5b2d0 namespace: kserve-ci-e2e-test resourceVersion: "22043" uid: c7ed41a0-1c1f-4202-a155-a79ef3eee9c1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-53302} kind: Pod name: ensemble-graph-53302-75ff947f68-lkcrm namespace: kserve-ci-e2e-test resourceVersion: "21997" uid: 683dbf0e-ba1a-45e7-83a1-28813d7c4bdc kind: Event lastTimestamp: "2026-07-20T10:21:57Z" message: 'Created container: ensemble-graph-53302' metadata: creationTimestamp: "2026-07-20T10:21:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:21:57Z" name: ensemble-graph-53302-75ff947f68-lkcrm.18c3f854389862f0 namespace: kserve-ci-e2e-test resourceVersion: "22044" uid: 108873df-dc0c-44af-adee-1c5c82486ddd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-53302} kind: Pod name: ensemble-graph-53302-75ff947f68-lkcrm namespace: kserve-ci-e2e-test resourceVersion: "21997" uid: 683dbf0e-ba1a-45e7-83a1-28813d7c4bdc kind: Event lastTimestamp: "2026-07-20T10:21:57Z" message: Started container ensemble-graph-53302 metadata: creationTimestamp: "2026-07-20T10:21:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:21:57Z" name: ensemble-graph-53302-75ff947f68-lkcrm.18c3f85439086aaf namespace: kserve-ci-e2e-test resourceVersion: "22045" uid: 06e52d1b-54a5-4f21-ad98-783f29d147df reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:30:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-53302} kind: Pod name: ensemble-graph-53302-75ff947f68-lkcrm namespace: kserve-ci-e2e-test resourceVersion: "21997" uid: 683dbf0e-ba1a-45e7-83a1-28813d7c4bdc kind: Event lastTimestamp: "2026-07-20T10:30:11Z" message: Stopping container ensemble-graph-53302 metadata: creationTimestamp: "2026-07-20T10:30:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:30:11Z" name: ensemble-graph-53302-75ff947f68-lkcrm.18c3f8c72219298a namespace: kserve-ci-e2e-test resourceVersion: "25874" uid: 88a888ca-3688-413e-9e2d-57a988ac50cc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-20T10:30:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-53302} kind: Pod name: ensemble-graph-53302-75ff947f68-lkcrm namespace: kserve-ci-e2e-test resourceVersion: "21997" uid: 683dbf0e-ba1a-45e7-83a1-28813d7c4bdc kind: Event lastTimestamp: "2026-07-20T10:30:39Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-20T10:30:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:30:39Z" name: ensemble-graph-53302-75ff947f68-lkcrm.18c3f8c7c6f8e90c namespace: kserve-ci-e2e-test resourceVersion: "26219" uid: 06d3427a-70e4-4ed8-be17-51bb3bdbf950 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-53302-75ff947f68 namespace: kserve-ci-e2e-test resourceVersion: "21992" uid: eeacf29f-21c5-4a62-a16a-498bbd0675fd kind: Event lastTimestamp: "2026-07-20T10:21:56Z" message: 'Created pod: ensemble-graph-53302-75ff947f68-lkcrm' metadata: creationTimestamp: "2026-07-20T10:21:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:21:56Z" name: ensemble-graph-53302-75ff947f68.18c3f853f6f652b1 namespace: kserve-ci-e2e-test resourceVersion: "21998" uid: 35746a42-5fd1-4023-9bf6-a30c36128c9c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-53302 namespace: kserve-ci-e2e-test resourceVersion: "21990" uid: 0006f7e7-f790-414e-939c-865a7814b030 kind: Event lastTimestamp: "2026-07-20T10:21:56Z" message: Scaled up replica set ensemble-graph-53302-75ff947f68 from 0 to 1 metadata: creationTimestamp: "2026-07-20T10:21:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:21:56Z" name: ensemble-graph-53302.18c3f853f60a5c50 namespace: kserve-ci-e2e-test resourceVersion: "21993" uid: 49ef7eed-c9b1-47c4-b840-c9aa7c621c0f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:56Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-53302 namespace: kserve-ci-e2e-test resourceVersion: "22011" uid: 42eea6d4-fbba-4fc6-bec0-24bb2797bea9 kind: Event lastTimestamp: "2026-07-20T10:21:56Z" message: InferenceGraph [ensemble-graph-53302] is Ready metadata: creationTimestamp: "2026-07-20T10:21:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:21:56Z" name: ensemble-graph-53302.18c3f853fb748172 namespace: kserve-ci-e2e-test resourceVersion: "22013" uid: f7a62278-eb26-4cb2-b290-0eb8fb53eaa3 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:56Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-53302 namespace: kserve-ci-e2e-test resourceVersion: "21982" uid: 42eea6d4-fbba-4fc6-bec0-24bb2797bea9 kind: Event lastTimestamp: "2026-07-20T10:21:56Z" message: 'Failed to update status for InferenceGraph "ensemble-graph-53302": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "ensemble-graph-53302": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-20T10:21:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:21:56Z" name: ensemble-graph-53302.18c3f853fd69830f namespace: kserve-ci-e2e-test resourceVersion: "22015" uid: dd4101e1-0ec1-4bc2-b517-b7d1164b56f4 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:56Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-53302 namespace: kserve-ci-e2e-test resourceVersion: "21982" uid: 42eea6d4-fbba-4fc6-bec0-24bb2797bea9 kind: Event lastTimestamp: "2026-07-20T10:21:56Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "ensemble-graph-53302": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-20T10:21:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:21:56Z" name: ensemble-graph-53302.18c3f853fd69c9a5 namespace: kserve-ci-e2e-test resourceVersion: "22016" uid: ec58530c-9c84-4044-a884-5756dff821e8 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-20T10:22:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-53302 namespace: kserve-ci-e2e-test resourceVersion: "21991" uid: cafc99d4-8359-41a5-bb99-7498ba822921 kind: Event lastTimestamp: "2026-07-20T10:22:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:22:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:22:56Z" name: ensemble-graph-53302.18c3f857747df3f8 namespace: kserve-ci-e2e-test resourceVersion: "22574" uid: 8a9c458f-d473-48b0-8ad8-74bc4654a2fe reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-20T10:22:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-53302 namespace: kserve-ci-e2e-test resourceVersion: "21991" uid: cafc99d4-8359-41a5-bb99-7498ba822921 kind: Event lastTimestamp: "2026-07-20T10:22:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:22:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:22:56Z" name: ensemble-graph-53302.18c3f85774ea67a1 namespace: kserve-ci-e2e-test resourceVersion: "22575" uid: b3011a6b-7c10-47a1-9ea4-a29ed3652f68 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-20T10:19:46.718497Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-242a5-predictor-656d487bbb-dhrhq namespace: kserve-ci-e2e-test resourceVersion: "20361" uid: b9eef044-e12a-4b03-91f4-06868c830382 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-242a5-predictor-656d487bbb-dhrhq to ip-10-0-143-155.ec2.internal metadata: creationTimestamp: "2026-07-20T10:19:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-20T10:19:46Z" name: error-404-isvc-242a5-predictor-656d487bbb-dhrhq.18c3f835a8d7a4c9 namespace: kserve-ci-e2e-test resourceVersion: "20368" uid: a569b344-4f8d-4056-8f43-de4d3a2995ee reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5dc669d9-69hkz source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:19:47Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-242a5-predictor-656d487bbb-dhrhq namespace: kserve-ci-e2e-test resourceVersion: "20374" uid: b9eef044-e12a-4b03-91f4-06868c830382 kind: Event lastTimestamp: "2026-07-20T10:19:47Z" message: Add eth0 [10.133.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-20T10:19:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-20T10:19:47Z" name: error-404-isvc-242a5-predictor-656d487bbb-dhrhq.18c3f835c310a55c namespace: kserve-ci-e2e-test resourceVersion: "20402" uid: 0a4d8c30-619d-4fa4-a81f-64db4e21eb1c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:19:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-242a5-predictor-656d487bbb-dhrhq namespace: kserve-ci-e2e-test resourceVersion: "20364" uid: b9eef044-e12a-4b03-91f4-06868c830382 kind: Event lastTimestamp: "2026-07-20T10:19:47Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1771" already present on machine metadata: creationTimestamp: "2026-07-20T10:19:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:19:47Z" name: error-404-isvc-242a5-predictor-656d487bbb-dhrhq.18c3f835c6ad1943 namespace: kserve-ci-e2e-test resourceVersion: "20407" uid: cb426a78-157d-44c9-8cab-ef8c799b4d24 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:19:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-242a5-predictor-656d487bbb-dhrhq namespace: kserve-ci-e2e-test resourceVersion: "20364" uid: b9eef044-e12a-4b03-91f4-06868c830382 kind: Event lastTimestamp: "2026-07-20T10:19:47Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-20T10:19:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:19:47Z" name: error-404-isvc-242a5-predictor-656d487bbb-dhrhq.18c3f835c906419e namespace: kserve-ci-e2e-test resourceVersion: "20411" uid: f89f7027-81c5-4287-80f9-37a05a0277d8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:19:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-242a5-predictor-656d487bbb-dhrhq namespace: kserve-ci-e2e-test resourceVersion: "20364" uid: b9eef044-e12a-4b03-91f4-06868c830382 kind: Event lastTimestamp: "2026-07-20T10:19:47Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-20T10:19:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:19:47Z" name: error-404-isvc-242a5-predictor-656d487bbb-dhrhq.18c3f835c98e820f namespace: kserve-ci-e2e-test resourceVersion: "20412" uid: 76ee8ab2-428a-4d28-8231-db048b3d1c4b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:19:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-242a5-predictor-656d487bbb-dhrhq namespace: kserve-ci-e2e-test resourceVersion: "20364" uid: b9eef044-e12a-4b03-91f4-06868c830382 kind: Event lastTimestamp: "2026-07-20T10:19:47Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-20T10:19:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:19:47Z" name: error-404-isvc-242a5-predictor-656d487bbb-dhrhq.18c3f835c99af928 namespace: kserve-ci-e2e-test resourceVersion: "20413" uid: cc9504c5-eb08-4ac6-a76c-26844752a648 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:19:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-242a5-predictor-656d487bbb-dhrhq namespace: kserve-ci-e2e-test resourceVersion: "20364" uid: b9eef044-e12a-4b03-91f4-06868c830382 kind: Event lastTimestamp: "2026-07-20T10:19:47Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-20T10:19:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:19:47Z" name: error-404-isvc-242a5-predictor-656d487bbb-dhrhq.18c3f835cc770447 namespace: kserve-ci-e2e-test resourceVersion: "20416" uid: b4461741-df24-4973-99c3-ffe6aef9424c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:19:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-242a5-predictor-656d487bbb-dhrhq namespace: kserve-ci-e2e-test resourceVersion: "20364" uid: b9eef044-e12a-4b03-91f4-06868c830382 kind: Event lastTimestamp: "2026-07-20T10:19:47Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:19:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:19:47Z" name: error-404-isvc-242a5-predictor-656d487bbb-dhrhq.18c3f835cce5252e namespace: kserve-ci-e2e-test resourceVersion: "20417" uid: 7fa277f4-67d7-41b4-b08c-59da3b6b74d8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-20T10:19:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-242a5-predictor-656d487bbb-dhrhq namespace: kserve-ci-e2e-test resourceVersion: "20364" uid: b9eef044-e12a-4b03-91f4-06868c830382 kind: Event lastTimestamp: "2026-07-20T10:20:23Z" message: 'Readiness probe failed: dial tcp 10.133.0.43:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-20T10:19:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:20:23Z" name: error-404-isvc-242a5-predictor-656d487bbb-dhrhq.18c3f835d4a65504 namespace: kserve-ci-e2e-test resourceVersion: "20712" uid: d6f8c60d-d295-4994-81d9-bb96ba32cca3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-242a5-predictor-656d487bbb-dhrhq namespace: kserve-ci-e2e-test resourceVersion: "20364" uid: b9eef044-e12a-4b03-91f4-06868c830382 kind: Event lastTimestamp: "2026-07-20T10:20:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-20T10:20:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:20:56Z" name: error-404-isvc-242a5-predictor-656d487bbb-dhrhq.18c3f845f514118f namespace: kserve-ci-e2e-test resourceVersion: "21208" uid: 23df040d-2822-4508-8a03-b7d27d0f15d4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-242a5-predictor-656d487bbb-dhrhq namespace: kserve-ci-e2e-test resourceVersion: "20364" uid: b9eef044-e12a-4b03-91f4-06868c830382 kind: Event lastTimestamp: "2026-07-20T10:20:56Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:20:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:20:56Z" name: error-404-isvc-242a5-predictor-656d487bbb-dhrhq.18c3f845f5147ee8 namespace: kserve-ci-e2e-test resourceVersion: "21210" uid: a5fc510e-b590-46b8-bc6f-e1be5883ba20 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-242a5-predictor-656d487bbb-dhrhq namespace: kserve-ci-e2e-test resourceVersion: "20364" uid: b9eef044-e12a-4b03-91f4-06868c830382 kind: Event lastTimestamp: "2026-07-20T10:20:58Z" message: 'Readiness probe failed: Get "https://10.133.0.43:8643/healthz": dial tcp 10.133.0.43:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-20T10:20:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:20:58Z" name: error-404-isvc-242a5-predictor-656d487bbb-dhrhq.18c3f8465d3f7dfd namespace: kserve-ci-e2e-test resourceVersion: "21326" uid: 1ac38065-e88f-4353-9bfb-0622d9a4293a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:19:46Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-242a5-predictor-656d487bbb namespace: kserve-ci-e2e-test resourceVersion: "20349" uid: 386d0c12-8bd5-4989-b2d0-c3909f7224d2 kind: Event lastTimestamp: "2026-07-20T10:19:46Z" message: 'Created pod: error-404-isvc-242a5-predictor-656d487bbb-dhrhq' metadata: creationTimestamp: "2026-07-20T10:19:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:19:46Z" name: error-404-isvc-242a5-predictor-656d487bbb.18c3f835a83ce008 namespace: kserve-ci-e2e-test resourceVersion: "20365" uid: d9445c09-4ab1-40d0-a00e-c63183f5601a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:19:46Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-242a5-predictor namespace: kserve-ci-e2e-test resourceVersion: "20338" uid: 25c8535e-a947-4406-989c-e500cfc150c2 kind: Event lastTimestamp: "2026-07-20T10:19:46Z" message: Scaled up replica set error-404-isvc-242a5-predictor-656d487bbb from 0 to 1 metadata: creationTimestamp: "2026-07-20T10:19:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:19:46Z" name: error-404-isvc-242a5-predictor.18c3f835a57f841c namespace: kserve-ci-e2e-test resourceVersion: "20353" uid: 8f9cd6cc-74ab-450e-8df6-371506822dfb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-20T10:20:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-242a5-predictor namespace: kserve-ci-e2e-test resourceVersion: "20346" uid: a6321662-bff2-45e4-9a05-5016a3520a69 kind: Event lastTimestamp: "2026-07-20T10:20:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:20:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:20:31Z" name: error-404-isvc-242a5-predictor.18c3f83924398647 namespace: kserve-ci-e2e-test resourceVersion: "20768" uid: 43ee500e-e52d-4d23-b272-5add1d3c4634 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-20T10:20:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-242a5-predictor namespace: kserve-ci-e2e-test resourceVersion: "20346" uid: a6321662-bff2-45e4-9a05-5016a3520a69 kind: Event lastTimestamp: "2026-07-20T10:20:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:20:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:20:31Z" name: error-404-isvc-242a5-predictor.18c3f83924a908fe namespace: kserve-ci-e2e-test resourceVersion: "20769" uid: e0fb0dbd-78d8-4b51-8bda-df7eeb5fa6a0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-242a5-predictor namespace: kserve-ci-e2e-test resourceVersion: "20549" uid: a6321662-bff2-45e4-9a05-5016a3520a69 kind: Event lastTimestamp: "2026-07-20T10:20:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:20:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:20:46Z" name: error-404-isvc-242a5-predictor.18c3f843a156a519 namespace: kserve-ci-e2e-test resourceVersion: "21076" uid: ef4d43d2-235f-418b-b864-6ac44a019be5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-242a5-predictor namespace: kserve-ci-e2e-test resourceVersion: "20549" uid: a6321662-bff2-45e4-9a05-5016a3520a69 kind: Event lastTimestamp: "2026-07-20T10:20:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:20:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:20:46Z" name: error-404-isvc-242a5-predictor.18c3f843a1c40b60 namespace: kserve-ci-e2e-test resourceVersion: "21078" uid: 5c4b5966-106c-4af2-aadc-db4121aa9208 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-20T10:19:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-242a5 namespace: kserve-ci-e2e-test resourceVersion: "20322" uid: b1d38250-4c7e-44bd-90b3-1098c6a16b26 kind: Event lastTimestamp: "2026-07-20T10:20:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-20T10:19:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:20:33Z" name: error-404-isvc-242a5.18c3f835a5d55537 namespace: kserve-ci-e2e-test resourceVersion: "20799" uid: a847ecb5-d6f1-428e-a8c3-f477ea926912 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:19:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-242a5 namespace: kserve-ci-e2e-test resourceVersion: "20322" uid: b1d38250-4c7e-44bd-90b3-1098c6a16b26 kind: Event lastTimestamp: "2026-07-20T10:19:46Z" message: 'Failed to update status for InferenceService "error-404-isvc-242a5": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-242a5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-20T10:19:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:19:46Z" name: error-404-isvc-242a5.18c3f835a6e46aea namespace: kserve-ci-e2e-test resourceVersion: "20358" uid: 12ca7134-4e15-4809-bb48-e0fe22bad22c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:19:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-242a5 namespace: kserve-ci-e2e-test resourceVersion: "20322" uid: b1d38250-4c7e-44bd-90b3-1098c6a16b26 kind: Event lastTimestamp: "2026-07-20T10:19:46Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-242a5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-20T10:19:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:19:46Z" name: error-404-isvc-242a5.18c3f835a6e4bd24 namespace: kserve-ci-e2e-test resourceVersion: "20362" uid: f890b0e8-a8c3-486f-bed0-a51e677d2308 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-242a5 namespace: kserve-ci-e2e-test resourceVersion: "20794" uid: b1d38250-4c7e-44bd-90b3-1098c6a16b26 kind: Event lastTimestamp: "2026-07-20T10:20:33Z" message: InferenceService [error-404-isvc-242a5] is Ready metadata: creationTimestamp: "2026-07-20T10:20:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:20:33Z" name: error-404-isvc-242a5.18c3f84092204cd3 namespace: kserve-ci-e2e-test resourceVersion: "20796" uid: 64668402-97f1-4bb9-a78f-460f031b9e1e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-20T10:20:36.328998Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-24b48-predictor-74c9c85698-qg4h5 namespace: kserve-ci-e2e-test resourceVersion: "20911" uid: 67383b80-8bd0-4351-b085-1ed83f615709 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-24b48-predictor-74c9c85698-qg4h5 to ip-10-0-143-155.ec2.internal metadata: creationTimestamp: "2026-07-20T10:20:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-20T10:20:36Z" name: error-404-isvc-24b48-predictor-74c9c85698-qg4h5.18c3f84135dbbcee namespace: kserve-ci-e2e-test resourceVersion: "20915" uid: 2a5ba2d0-d834-4d02-8e8a-383fda9938ac reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5dc669d9-69hkz source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:36Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-24b48-predictor-74c9c85698-qg4h5 namespace: kserve-ci-e2e-test resourceVersion: "20913" uid: 67383b80-8bd0-4351-b085-1ed83f615709 kind: Event lastTimestamp: "2026-07-20T10:20:36Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-24b48-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-20T10:20:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:20:36Z" name: error-404-isvc-24b48-predictor-74c9c85698-qg4h5.18c3f84141450064 namespace: kserve-ci-e2e-test resourceVersion: "20932" uid: e7abe68f-f146-4df9-97ed-b8ef42ef0020 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:37Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-24b48-predictor-74c9c85698-qg4h5 namespace: kserve-ci-e2e-test resourceVersion: "20921" uid: 67383b80-8bd0-4351-b085-1ed83f615709 kind: Event lastTimestamp: "2026-07-20T10:20:37Z" message: Add eth0 [10.133.0.45/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-20T10:20:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-20T10:20:37Z" name: error-404-isvc-24b48-predictor-74c9c85698-qg4h5.18c3f841749503bf namespace: kserve-ci-e2e-test resourceVersion: "20947" uid: e7c5762a-87a5-4713-807d-db8193559658 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-24b48-predictor-74c9c85698-qg4h5 namespace: kserve-ci-e2e-test resourceVersion: "20913" uid: 67383b80-8bd0-4351-b085-1ed83f615709 kind: Event lastTimestamp: "2026-07-20T10:20:37Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1771" already present on machine metadata: creationTimestamp: "2026-07-20T10:20:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:20:37Z" name: error-404-isvc-24b48-predictor-74c9c85698-qg4h5.18c3f8417649f2b2 namespace: kserve-ci-e2e-test resourceVersion: "20949" uid: 15e116db-b5e1-4fb7-aa35-7097ab1eb071 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-24b48-predictor-74c9c85698-qg4h5 namespace: kserve-ci-e2e-test resourceVersion: "20913" uid: 67383b80-8bd0-4351-b085-1ed83f615709 kind: Event lastTimestamp: "2026-07-20T10:20:37Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-20T10:20:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:20:37Z" name: error-404-isvc-24b48-predictor-74c9c85698-qg4h5.18c3f84179c4de03 namespace: kserve-ci-e2e-test resourceVersion: "20952" uid: 47bc054a-854b-4f7d-9080-3f82c578159b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-24b48-predictor-74c9c85698-qg4h5 namespace: kserve-ci-e2e-test resourceVersion: "20913" uid: 67383b80-8bd0-4351-b085-1ed83f615709 kind: Event lastTimestamp: "2026-07-20T10:20:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-20T10:20:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:20:37Z" name: error-404-isvc-24b48-predictor-74c9c85698-qg4h5.18c3f8417a327065 namespace: kserve-ci-e2e-test resourceVersion: "20953" uid: ad72de04-e3d7-4b45-9ff1-9cf0f80cc368 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-24b48-predictor-74c9c85698-qg4h5 namespace: kserve-ci-e2e-test resourceVersion: "20913" uid: 67383b80-8bd0-4351-b085-1ed83f615709 kind: Event lastTimestamp: "2026-07-20T10:20:37Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-20T10:20:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:20:37Z" name: error-404-isvc-24b48-predictor-74c9c85698-qg4h5.18c3f8417a3bf58f namespace: kserve-ci-e2e-test resourceVersion: "20954" uid: 6972e8e2-bfbd-4105-a6ef-1d1cc8ef219b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-24b48-predictor-74c9c85698-qg4h5 namespace: kserve-ci-e2e-test resourceVersion: "20913" uid: 67383b80-8bd0-4351-b085-1ed83f615709 kind: Event lastTimestamp: "2026-07-20T10:20:37Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-20T10:20:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:20:37Z" name: error-404-isvc-24b48-predictor-74c9c85698-qg4h5.18c3f8417d08ec7b namespace: kserve-ci-e2e-test resourceVersion: "20956" uid: 46a4bcf1-e7ae-44fe-a99c-256c2172c691 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-24b48-predictor-74c9c85698-qg4h5 namespace: kserve-ci-e2e-test resourceVersion: "20913" uid: 67383b80-8bd0-4351-b085-1ed83f615709 kind: Event lastTimestamp: "2026-07-20T10:20:37Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:20:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:20:37Z" name: error-404-isvc-24b48-predictor-74c9c85698-qg4h5.18c3f8417d767618 namespace: kserve-ci-e2e-test resourceVersion: "20957" uid: 6cff5bd2-b185-41a7-9baa-fe980702c571 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-20T10:20:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-24b48-predictor-74c9c85698-qg4h5 namespace: kserve-ci-e2e-test resourceVersion: "20913" uid: 67383b80-8bd0-4351-b085-1ed83f615709 kind: Event lastTimestamp: "2026-07-20T10:21:23Z" message: 'Readiness probe failed: dial tcp 10.133.0.45:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-20T10:20:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:21:23Z" name: error-404-isvc-24b48-predictor-74c9c85698-qg4h5.18c3f84188d78fb0 namespace: kserve-ci-e2e-test resourceVersion: "21540" uid: 57dbe031-69a5-4879-9c0b-15ac019697c1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-24b48-predictor-74c9c85698-qg4h5 namespace: kserve-ci-e2e-test resourceVersion: "20913" uid: 67383b80-8bd0-4351-b085-1ed83f615709 kind: Event lastTimestamp: "2026-07-20T10:21:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-20T10:21:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:21:56Z" name: error-404-isvc-24b48-predictor-74c9c85698-qg4h5.18c3f853df4fe996 namespace: kserve-ci-e2e-test resourceVersion: "21924" uid: 4a861775-8264-48dc-b814-278ab88b2ba2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-24b48-predictor-74c9c85698-qg4h5 namespace: kserve-ci-e2e-test resourceVersion: "20913" uid: 67383b80-8bd0-4351-b085-1ed83f615709 kind: Event lastTimestamp: "2026-07-20T10:21:56Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:21:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:21:56Z" name: error-404-isvc-24b48-predictor-74c9c85698-qg4h5.18c3f853df51cbfc namespace: kserve-ci-e2e-test resourceVersion: "21927" uid: 6d976810-471a-4e4c-8a45-d41b02f28b19 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-24b48-predictor-74c9c85698-qg4h5 namespace: kserve-ci-e2e-test resourceVersion: "20913" uid: 67383b80-8bd0-4351-b085-1ed83f615709 kind: Event lastTimestamp: "2026-07-20T10:21:58Z" message: 'Readiness probe failed: Get "https://10.133.0.45:8643/healthz": dial tcp 10.133.0.45:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-20T10:21:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:21:58Z" name: error-404-isvc-24b48-predictor-74c9c85698-qg4h5.18c3f854651c7bc8 namespace: kserve-ci-e2e-test resourceVersion: "22061" uid: 04e6e235-cb06-479f-91e6-905ec1cae86c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-24b48-predictor-74c9c85698 namespace: kserve-ci-e2e-test resourceVersion: "20904" uid: a8bbdd3f-0e6d-40a7-b133-0e2409895bda kind: Event lastTimestamp: "2026-07-20T10:20:36Z" message: 'Created pod: error-404-isvc-24b48-predictor-74c9c85698-qg4h5' metadata: creationTimestamp: "2026-07-20T10:20:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:20:36Z" name: error-404-isvc-24b48-predictor-74c9c85698.18c3f8413570f0f2 namespace: kserve-ci-e2e-test resourceVersion: "20912" uid: da88967d-6ef5-4b05-95f6-0c863dfcb823 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-24b48-predictor namespace: kserve-ci-e2e-test resourceVersion: "20902" uid: 2f43eb67-c6ed-4b98-b0ba-c4d04f73ccc3 kind: Event lastTimestamp: "2026-07-20T10:20:36Z" message: Scaled up replica set error-404-isvc-24b48-predictor-74c9c85698 from 0 to 1 metadata: creationTimestamp: "2026-07-20T10:20:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:20:36Z" name: error-404-isvc-24b48-predictor.18c3f84133d8c2ec namespace: kserve-ci-e2e-test resourceVersion: "20905" uid: 76cd1c75-e7fc-4887-8cc8-8c566e9b95a5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-24b48-predictor namespace: kserve-ci-e2e-test resourceVersion: "20903" uid: 623a50cc-a65c-4386-b4b2-1477c7fc9ef8 kind: Event lastTimestamp: "2026-07-20T10:20:51Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:20:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:20:51Z" name: error-404-isvc-24b48-predictor.18c3f844b2e8f69e namespace: kserve-ci-e2e-test resourceVersion: "21130" uid: 91c3f03f-3310-4fbf-83db-04fed93e258b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-24b48-predictor namespace: kserve-ci-e2e-test resourceVersion: "20903" uid: 623a50cc-a65c-4386-b4b2-1477c7fc9ef8 kind: Event lastTimestamp: "2026-07-20T10:20:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:20:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:20:51Z" name: error-404-isvc-24b48-predictor.18c3f844b3d8cd45 namespace: kserve-ci-e2e-test resourceVersion: "21131" uid: 93853e61-7f61-4755-9ec6-84e7260b4c2f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-20T10:21:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-24b48-predictor namespace: kserve-ci-e2e-test resourceVersion: "21129" uid: 623a50cc-a65c-4386-b4b2-1477c7fc9ef8 kind: Event lastTimestamp: "2026-07-20T10:21:51Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:21:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:21:51Z" name: error-404-isvc-24b48-predictor.18c3f84832583608 namespace: kserve-ci-e2e-test resourceVersion: "21826" uid: 944f4aba-68bd-4724-b66b-e71799549d07 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-20T10:21:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-24b48-predictor namespace: kserve-ci-e2e-test resourceVersion: "21129" uid: 623a50cc-a65c-4386-b4b2-1477c7fc9ef8 kind: Event lastTimestamp: "2026-07-20T10:21:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:21:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:21:51Z" name: error-404-isvc-24b48-predictor.18c3f84832d0f5dd namespace: kserve-ci-e2e-test resourceVersion: "21827" uid: f54ccb09-cbdd-4a0e-9a8d-e75294e576ae reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-20T10:20:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-24b48 namespace: kserve-ci-e2e-test resourceVersion: "20894" uid: 42a52734-184a-413e-9b07-43c8c96c3288 kind: Event lastTimestamp: "2026-07-20T10:21:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-20T10:20:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:21:33Z" name: error-404-isvc-24b48.18c3f84133d87ebf namespace: kserve-ci-e2e-test resourceVersion: "21638" uid: b96b2a02-89d8-40e6-ad1c-4ee26bfacc77 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-24b48 namespace: kserve-ci-e2e-test resourceVersion: "20894" uid: 42a52734-184a-413e-9b07-43c8c96c3288 kind: Event lastTimestamp: "2026-07-20T10:20:36Z" message: 'Failed to update status for InferenceService "error-404-isvc-24b48": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-24b48": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-20T10:20:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:20:36Z" name: error-404-isvc-24b48.18c3f841346dc833 namespace: kserve-ci-e2e-test resourceVersion: "20908" uid: a65463cf-6dc1-4554-b0bf-aec1db08befa reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-24b48 namespace: kserve-ci-e2e-test resourceVersion: "20894" uid: 42a52734-184a-413e-9b07-43c8c96c3288 kind: Event lastTimestamp: "2026-07-20T10:20:36Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-24b48": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-20T10:20:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:20:36Z" name: error-404-isvc-24b48.18c3f841346e0d9d namespace: kserve-ci-e2e-test resourceVersion: "20909" uid: 865e8391-ad59-4e22-a88b-3a177ef1c423 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-24b48 namespace: kserve-ci-e2e-test resourceVersion: "21631" uid: 42a52734-184a-413e-9b07-43c8c96c3288 kind: Event lastTimestamp: "2026-07-20T10:21:33Z" message: InferenceService [error-404-isvc-24b48] is Ready metadata: creationTimestamp: "2026-07-20T10:21:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:21:33Z" name: error-404-isvc-24b48.18c3f84e999ecff2 namespace: kserve-ci-e2e-test resourceVersion: "21632" uid: bb15d52d-4c07-4166-a684-039098081c44 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-20T10:20:56.911839Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-53302-predictor-5b4fdf4964-w5cwv namespace: kserve-ci-e2e-test resourceVersion: "21271" uid: 4c0648a8-63bf-4a92-8017-6921e4d12346 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-53302-predictor-5b4fdf4964-w5cwv to ip-10-0-143-155.ec2.internal metadata: creationTimestamp: "2026-07-20T10:20:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-20T10:20:56Z" name: error-404-isvc-53302-predictor-5b4fdf4964-w5cwv.18c3f84600b10187 namespace: kserve-ci-e2e-test resourceVersion: "21275" uid: 846b79be-d594-48f1-a1d3-4b40add9d31f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5dc669d9-69hkz source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:57Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-53302-predictor-5b4fdf4964-w5cwv namespace: kserve-ci-e2e-test resourceVersion: "21281" uid: 4c0648a8-63bf-4a92-8017-6921e4d12346 kind: Event lastTimestamp: "2026-07-20T10:20:57Z" message: Add eth0 [10.133.0.47/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-20T10:20:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-20T10:20:57Z" name: error-404-isvc-53302-predictor-5b4fdf4964-w5cwv.18c3f8461b07bd98 namespace: kserve-ci-e2e-test resourceVersion: "21296" uid: 06b9cf19-2a3a-4ccb-803b-638d3e304957 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-53302-predictor-5b4fdf4964-w5cwv namespace: kserve-ci-e2e-test resourceVersion: "21274" uid: 4c0648a8-63bf-4a92-8017-6921e4d12346 kind: Event lastTimestamp: "2026-07-20T10:20:57Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1771" already present on machine metadata: creationTimestamp: "2026-07-20T10:20:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:20:57Z" name: error-404-isvc-53302-predictor-5b4fdf4964-w5cwv.18c3f8461c6f0a37 namespace: kserve-ci-e2e-test resourceVersion: "21298" uid: 0f55b4ef-4123-4027-8dbc-9639fbe667d0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-53302-predictor-5b4fdf4964-w5cwv namespace: kserve-ci-e2e-test resourceVersion: "21274" uid: 4c0648a8-63bf-4a92-8017-6921e4d12346 kind: Event lastTimestamp: "2026-07-20T10:20:57Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-20T10:20:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:20:57Z" name: error-404-isvc-53302-predictor-5b4fdf4964-w5cwv.18c3f8461e83c515 namespace: kserve-ci-e2e-test resourceVersion: "21301" uid: 7c8b2757-0b01-452b-9b8b-59847196a58a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-53302-predictor-5b4fdf4964-w5cwv namespace: kserve-ci-e2e-test resourceVersion: "21274" uid: 4c0648a8-63bf-4a92-8017-6921e4d12346 kind: Event lastTimestamp: "2026-07-20T10:20:57Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-20T10:20:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:20:57Z" name: error-404-isvc-53302-predictor-5b4fdf4964-w5cwv.18c3f8461efeb619 namespace: kserve-ci-e2e-test resourceVersion: "21302" uid: 13698304-9dff-4f5d-8932-1f443a7bb402 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-53302-predictor-5b4fdf4964-w5cwv namespace: kserve-ci-e2e-test resourceVersion: "21274" uid: 4c0648a8-63bf-4a92-8017-6921e4d12346 kind: Event lastTimestamp: "2026-07-20T10:20:57Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-20T10:20:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:20:57Z" name: error-404-isvc-53302-predictor-5b4fdf4964-w5cwv.18c3f8461f07456f namespace: kserve-ci-e2e-test resourceVersion: "21303" uid: ddd4a174-8469-436e-8bb7-eea0745bbc07 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-53302-predictor-5b4fdf4964-w5cwv namespace: kserve-ci-e2e-test resourceVersion: "21274" uid: 4c0648a8-63bf-4a92-8017-6921e4d12346 kind: Event lastTimestamp: "2026-07-20T10:20:57Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-20T10:20:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:20:57Z" name: error-404-isvc-53302-predictor-5b4fdf4964-w5cwv.18c3f84621d94dc4 namespace: kserve-ci-e2e-test resourceVersion: "21305" uid: 83410183-096f-49fa-81d9-8f3a6d20689c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-53302-predictor-5b4fdf4964-w5cwv namespace: kserve-ci-e2e-test resourceVersion: "21274" uid: 4c0648a8-63bf-4a92-8017-6921e4d12346 kind: Event lastTimestamp: "2026-07-20T10:20:57Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:20:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:20:57Z" name: error-404-isvc-53302-predictor-5b4fdf4964-w5cwv.18c3f846224dd879 namespace: kserve-ci-e2e-test resourceVersion: "21306" uid: a3c46bd2-1666-4fa6-8b11-c012159b5cb0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-20T10:20:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-53302-predictor-5b4fdf4964-w5cwv namespace: kserve-ci-e2e-test resourceVersion: "21274" uid: 4c0648a8-63bf-4a92-8017-6921e4d12346 kind: Event lastTimestamp: "2026-07-20T10:30:13Z" message: 'Readiness probe failed: dial tcp 10.133.0.47:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-20T10:20:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:30:13Z" name: error-404-isvc-53302-predictor-5b4fdf4964-w5cwv.18c3f84637742dbe namespace: kserve-ci-e2e-test resourceVersion: "26018" uid: ff05e3d9-5596-498a-95ca-5d797073c768 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:30:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-53302-predictor-5b4fdf4964-w5cwv namespace: kserve-ci-e2e-test resourceVersion: "21274" uid: 4c0648a8-63bf-4a92-8017-6921e4d12346 kind: Event lastTimestamp: "2026-07-20T10:30:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-20T10:30:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:30:11Z" name: error-404-isvc-53302-predictor-5b4fdf4964-w5cwv.18c3f8c73cea36bf namespace: kserve-ci-e2e-test resourceVersion: "25969" uid: 210ca147-8af1-46ff-aa04-91399516cc11 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:30:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-53302-predictor-5b4fdf4964-w5cwv namespace: kserve-ci-e2e-test resourceVersion: "21274" uid: 4c0648a8-63bf-4a92-8017-6921e4d12346 kind: Event lastTimestamp: "2026-07-20T10:30:11Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:30:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:30:11Z" name: error-404-isvc-53302-predictor-5b4fdf4964-w5cwv.18c3f8c73ceac02d namespace: kserve-ci-e2e-test resourceVersion: "25970" uid: 5412491b-e497-410b-aac1-499e08c31436 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:30:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-53302-predictor-5b4fdf4964-w5cwv namespace: kserve-ci-e2e-test resourceVersion: "21274" uid: 4c0648a8-63bf-4a92-8017-6921e4d12346 kind: Event lastTimestamp: "2026-07-20T10:30:13Z" message: 'Readiness probe failed: Get "https://10.133.0.47:8643/healthz": dial tcp 10.133.0.47:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-20T10:30:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:30:13Z" name: error-404-isvc-53302-predictor-5b4fdf4964-w5cwv.18c3f8c7abe85cea namespace: kserve-ci-e2e-test resourceVersion: "26014" uid: 1cf1e6c1-b386-4b63-85ee-db73063f2982 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-53302-predictor-5b4fdf4964 namespace: kserve-ci-e2e-test resourceVersion: "21263" uid: 2dd1e3ab-f15c-4073-b173-32911dfe0fde kind: Event lastTimestamp: "2026-07-20T10:20:56Z" message: 'Created pod: error-404-isvc-53302-predictor-5b4fdf4964-w5cwv' metadata: creationTimestamp: "2026-07-20T10:20:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:20:56Z" name: error-404-isvc-53302-predictor-5b4fdf4964.18c3f846001664a7 namespace: kserve-ci-e2e-test resourceVersion: "21272" uid: 0922a70b-775b-464d-84ad-61a3073c0450 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-53302-predictor namespace: kserve-ci-e2e-test resourceVersion: "21260" uid: e56326c1-48d1-448a-a1bc-e96b84ad6622 kind: Event lastTimestamp: "2026-07-20T10:20:56Z" message: Scaled up replica set error-404-isvc-53302-predictor-5b4fdf4964 from 0 to 1 metadata: creationTimestamp: "2026-07-20T10:20:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:20:56Z" name: error-404-isvc-53302-predictor.18c3f845fe5e680d namespace: kserve-ci-e2e-test resourceVersion: "21265" uid: 7e2dd599-1a24-4e6b-8503-3a2a7aa269a5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-20T10:21:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-53302-predictor namespace: kserve-ci-e2e-test resourceVersion: "21262" uid: c27099af-923b-43c7-b412-8aef8dc4f90f kind: Event lastTimestamp: "2026-07-20T10:21:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:21:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:21:26Z" name: error-404-isvc-53302-predictor.18c3f8497d0afe36 namespace: kserve-ci-e2e-test resourceVersion: "21568" uid: 36d212ff-f806-459a-ac34-e281a086bc02 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-20T10:21:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-53302-predictor namespace: kserve-ci-e2e-test resourceVersion: "21262" uid: c27099af-923b-43c7-b412-8aef8dc4f90f kind: Event lastTimestamp: "2026-07-20T10:21:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:21:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:21:26Z" name: error-404-isvc-53302-predictor.18c3f8497d850fe5 namespace: kserve-ci-e2e-test resourceVersion: "21569" uid: 84229d8f-daa1-417b-b72d-3787107c9718 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-20T10:21:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-53302-predictor namespace: kserve-ci-e2e-test resourceVersion: "21448" uid: c27099af-923b-43c7-b412-8aef8dc4f90f kind: Event lastTimestamp: "2026-07-20T10:22:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:21:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:22:11Z" name: error-404-isvc-53302-predictor.18c3f8507ac7c179 namespace: kserve-ci-e2e-test resourceVersion: "22193" uid: 65e603cc-3b27-4504-98fd-f6cd35ab49dc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-20T10:21:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-53302-predictor namespace: kserve-ci-e2e-test resourceVersion: "21448" uid: c27099af-923b-43c7-b412-8aef8dc4f90f kind: Event lastTimestamp: "2026-07-20T10:22:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:21:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:22:11Z" name: error-404-isvc-53302-predictor.18c3f8507b2877f9 namespace: kserve-ci-e2e-test resourceVersion: "22194" uid: 362539fd-8213-473a-8433-9db021abbfc5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-20T10:20:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-53302 namespace: kserve-ci-e2e-test resourceVersion: "21243" uid: 9b721c77-c31f-4748-bee4-724dcfcdcb82 kind: Event lastTimestamp: "2026-07-20T10:21:44Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-20T10:20:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:21:44Z" name: error-404-isvc-53302.18c3f845fe6ef5d7 namespace: kserve-ci-e2e-test resourceVersion: "21742" uid: 561a07c2-f42d-46bd-a4cc-df04768f8736 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-53302 namespace: kserve-ci-e2e-test resourceVersion: "21243" uid: 9b721c77-c31f-4748-bee4-724dcfcdcb82 kind: Event lastTimestamp: "2026-07-20T10:20:56Z" message: 'Failed to update status for InferenceService "error-404-isvc-53302": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-53302": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-20T10:20:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:20:56Z" name: error-404-isvc-53302.18c3f845feeb9aa9 namespace: kserve-ci-e2e-test resourceVersion: "21268" uid: e64bcb02-6ba2-471e-9985-ed864abb48d1 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-53302 namespace: kserve-ci-e2e-test resourceVersion: "21243" uid: 9b721c77-c31f-4748-bee4-724dcfcdcb82 kind: Event lastTimestamp: "2026-07-20T10:20:56Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-53302": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-20T10:20:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:20:56Z" name: error-404-isvc-53302.18c3f845feebe8a2 namespace: kserve-ci-e2e-test resourceVersion: "21269" uid: e936b8b0-bead-47c3-a0b9-b08592c3fe78 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-53302 namespace: kserve-ci-e2e-test resourceVersion: "21737" uid: 9b721c77-c31f-4748-bee4-724dcfcdcb82 kind: Event lastTimestamp: "2026-07-20T10:21:44Z" message: InferenceService [error-404-isvc-53302] is Ready metadata: creationTimestamp: "2026-07-20T10:21:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:21:44Z" name: error-404-isvc-53302.18c3f850fc536185 namespace: kserve-ci-e2e-test resourceVersion: "21739" uid: 8f37a3d7-d951-4d12-b96c-ee363e2777a6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-20T10:21:56.628798Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-64e78-predictor-b57c5f466-7vqfm namespace: kserve-ci-e2e-test resourceVersion: "21961" uid: ecfbf234-83ae-48f9-bfbe-cefac206efc7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-64e78-predictor-b57c5f466-7vqfm to ip-10-0-143-155.ec2.internal metadata: creationTimestamp: "2026-07-20T10:21:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-20T10:21:56Z" name: error-404-isvc-64e78-predictor-b57c5f466-7vqfm.18c3f853e8196cdd namespace: kserve-ci-e2e-test resourceVersion: "21965" uid: 82804648-50bf-4c51-904e-023089b00b3c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5dc669d9-69hkz source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:57Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-64e78-predictor-b57c5f466-7vqfm namespace: kserve-ci-e2e-test resourceVersion: "21972" uid: ecfbf234-83ae-48f9-bfbe-cefac206efc7 kind: Event lastTimestamp: "2026-07-20T10:21:57Z" message: Add eth0 [10.133.0.49/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-20T10:21:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-20T10:21:57Z" name: error-404-isvc-64e78-predictor-b57c5f466-7vqfm.18c3f85401e32395 namespace: kserve-ci-e2e-test resourceVersion: "22018" uid: 6fc86f06-c814-4a20-8f8b-65960be3e636 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-64e78-predictor-b57c5f466-7vqfm namespace: kserve-ci-e2e-test resourceVersion: "21963" uid: ecfbf234-83ae-48f9-bfbe-cefac206efc7 kind: Event lastTimestamp: "2026-07-20T10:21:57Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1771" already present on machine metadata: creationTimestamp: "2026-07-20T10:21:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:21:57Z" name: error-404-isvc-64e78-predictor-b57c5f466-7vqfm.18c3f854032350c7 namespace: kserve-ci-e2e-test resourceVersion: "22021" uid: d8b881d2-ebf8-41bf-a55e-ad0a5dca9e30 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-64e78-predictor-b57c5f466-7vqfm namespace: kserve-ci-e2e-test resourceVersion: "21963" uid: ecfbf234-83ae-48f9-bfbe-cefac206efc7 kind: Event lastTimestamp: "2026-07-20T10:21:57Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-20T10:21:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:21:57Z" name: error-404-isvc-64e78-predictor-b57c5f466-7vqfm.18c3f8540701aeb6 namespace: kserve-ci-e2e-test resourceVersion: "22022" uid: bb123f64-6d4e-4584-99c4-0603c22fa9f0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-64e78-predictor-b57c5f466-7vqfm namespace: kserve-ci-e2e-test resourceVersion: "21963" uid: ecfbf234-83ae-48f9-bfbe-cefac206efc7 kind: Event lastTimestamp: "2026-07-20T10:21:57Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-20T10:21:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:21:57Z" name: error-404-isvc-64e78-predictor-b57c5f466-7vqfm.18c3f8540772accd namespace: kserve-ci-e2e-test resourceVersion: "22023" uid: 07e230b0-cf8e-438f-b9a0-93c10a2e33fb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-64e78-predictor-b57c5f466-7vqfm namespace: kserve-ci-e2e-test resourceVersion: "21963" uid: ecfbf234-83ae-48f9-bfbe-cefac206efc7 kind: Event lastTimestamp: "2026-07-20T10:21:57Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-20T10:21:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:21:57Z" name: error-404-isvc-64e78-predictor-b57c5f466-7vqfm.18c3f854077bd2e2 namespace: kserve-ci-e2e-test resourceVersion: "22024" uid: 783d43c3-754d-4998-8578-14f1d52e6fe7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-64e78-predictor-b57c5f466-7vqfm namespace: kserve-ci-e2e-test resourceVersion: "21963" uid: ecfbf234-83ae-48f9-bfbe-cefac206efc7 kind: Event lastTimestamp: "2026-07-20T10:21:57Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-20T10:21:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:21:57Z" name: error-404-isvc-64e78-predictor-b57c5f466-7vqfm.18c3f8540ab6d66e namespace: kserve-ci-e2e-test resourceVersion: "22025" uid: bc86b833-54a3-47a7-b58e-6c4b2e1bf877 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-64e78-predictor-b57c5f466-7vqfm namespace: kserve-ci-e2e-test resourceVersion: "21963" uid: ecfbf234-83ae-48f9-bfbe-cefac206efc7 kind: Event lastTimestamp: "2026-07-20T10:21:57Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:21:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:21:57Z" name: error-404-isvc-64e78-predictor-b57c5f466-7vqfm.18c3f8540b24f98e namespace: kserve-ci-e2e-test resourceVersion: "22026" uid: 5543ab3c-2ef8-47cc-86be-dd7156072085 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-20T10:21:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-64e78-predictor-b57c5f466-7vqfm namespace: kserve-ci-e2e-test resourceVersion: "21963" uid: ecfbf234-83ae-48f9-bfbe-cefac206efc7 kind: Event lastTimestamp: "2026-07-20T10:31:14Z" message: 'Readiness probe failed: dial tcp 10.133.0.49:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-20T10:21:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:31:14Z" name: error-404-isvc-64e78-predictor-b57c5f466-7vqfm.18c3f854422f5545 namespace: kserve-ci-e2e-test resourceVersion: "26686" uid: c2345075-f901-4b15-91c6-1d6dc3b79924 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-64e78-predictor-b57c5f466-7vqfm namespace: kserve-ci-e2e-test resourceVersion: "21963" uid: ecfbf234-83ae-48f9-bfbe-cefac206efc7 kind: Event lastTimestamp: "2026-07-20T10:31:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-20T10:31:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:31:11Z" name: error-404-isvc-64e78-predictor-b57c5f466-7vqfm.18c3f8d5089bf6b4 namespace: kserve-ci-e2e-test resourceVersion: "26512" uid: 0a2c0e39-c3e5-4393-be00-f3dc8b898818 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-64e78-predictor-b57c5f466-7vqfm namespace: kserve-ci-e2e-test resourceVersion: "21963" uid: ecfbf234-83ae-48f9-bfbe-cefac206efc7 kind: Event lastTimestamp: "2026-07-20T10:31:11Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:31:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:31:11Z" name: error-404-isvc-64e78-predictor-b57c5f466-7vqfm.18c3f8d5089c9388 namespace: kserve-ci-e2e-test resourceVersion: "26515" uid: 819eba13-4c67-4b7d-9106-84bd369c4658 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-64e78-predictor-b57c5f466-7vqfm namespace: kserve-ci-e2e-test resourceVersion: "21963" uid: ecfbf234-83ae-48f9-bfbe-cefac206efc7 kind: Event lastTimestamp: "2026-07-20T10:31:14Z" message: 'Readiness probe failed: Get "https://10.133.0.49:8643/healthz": dial tcp 10.133.0.49:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-20T10:31:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:31:14Z" name: error-404-isvc-64e78-predictor-b57c5f466-7vqfm.18c3f8d5b6b48e56 namespace: kserve-ci-e2e-test resourceVersion: "26685" uid: 87478007-9cb4-44a9-a101-ed93c7858ebc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-64e78-predictor-b57c5f466 namespace: kserve-ci-e2e-test resourceVersion: "21954" uid: 6450d383-2a1e-4e44-be28-78f81b14ad52 kind: Event lastTimestamp: "2026-07-20T10:21:56Z" message: 'Created pod: error-404-isvc-64e78-predictor-b57c5f466-7vqfm' metadata: creationTimestamp: "2026-07-20T10:21:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:21:56Z" name: error-404-isvc-64e78-predictor-b57c5f466.18c3f853e786a76e namespace: kserve-ci-e2e-test resourceVersion: "21962" uid: dae39c95-0388-4213-879e-a71a605fa43f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-64e78-predictor namespace: kserve-ci-e2e-test resourceVersion: "21951" uid: ed598531-6660-4750-8e15-74e50a22e4cc kind: Event lastTimestamp: "2026-07-20T10:21:56Z" message: Scaled up replica set error-404-isvc-64e78-predictor-b57c5f466 from 0 to 1 metadata: creationTimestamp: "2026-07-20T10:21:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:21:56Z" name: error-404-isvc-64e78-predictor.18c3f853e5dd24f6 namespace: kserve-ci-e2e-test resourceVersion: "21956" uid: b9b17993-5be0-43ef-be35-aaa6e0b81f19 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-20T10:22:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-64e78-predictor namespace: kserve-ci-e2e-test resourceVersion: "21953" uid: de2c92c7-0cfd-4afd-878e-1b942004941a kind: Event lastTimestamp: "2026-07-20T10:22:41Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:22:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:22:41Z" name: error-404-isvc-64e78-predictor.18c3f85764952dfe namespace: kserve-ci-e2e-test resourceVersion: "22416" uid: 749e5863-1c8b-4418-b49b-eba0edb13c01 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-20T10:22:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-64e78-predictor namespace: kserve-ci-e2e-test resourceVersion: "21953" uid: de2c92c7-0cfd-4afd-878e-1b942004941a kind: Event lastTimestamp: "2026-07-20T10:22:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:22:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:22:41Z" name: error-404-isvc-64e78-predictor.18c3f85765301b9b namespace: kserve-ci-e2e-test resourceVersion: "22417" uid: ca00ca3a-3d10-4da3-a14b-f427d70f8a8c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-20T10:22:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-64e78-predictor namespace: kserve-ci-e2e-test resourceVersion: "22183" uid: de2c92c7-0cfd-4afd-878e-1b942004941a kind: Event lastTimestamp: "2026-07-20T10:23:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:22:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:23:11Z" name: error-404-isvc-64e78-predictor.18c3f861e1ae5250 namespace: kserve-ci-e2e-test resourceVersion: "22694" uid: e813294e-2fb1-449d-8820-2569f867edb3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-20T10:22:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-64e78-predictor namespace: kserve-ci-e2e-test resourceVersion: "22183" uid: de2c92c7-0cfd-4afd-878e-1b942004941a kind: Event lastTimestamp: "2026-07-20T10:23:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:22:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:23:11Z" name: error-404-isvc-64e78-predictor.18c3f861e2349a4d namespace: kserve-ci-e2e-test resourceVersion: "22695" uid: 83c91cfb-b5fa-46c8-88ee-73cab5a6bdea reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-20T10:21:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-64e78 namespace: kserve-ci-e2e-test resourceVersion: "21937" uid: f2492e4c-8884-4f45-806c-89953c3d796d kind: Event lastTimestamp: "2026-07-20T10:22:44Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-20T10:21:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:22:44Z" name: error-404-isvc-64e78.18c3f853e5dfac69 namespace: kserve-ci-e2e-test resourceVersion: "22444" uid: 7df92a49-50ac-47f2-bedf-f4508139865d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-64e78 namespace: kserve-ci-e2e-test resourceVersion: "21937" uid: f2492e4c-8884-4f45-806c-89953c3d796d kind: Event lastTimestamp: "2026-07-20T10:21:56Z" message: 'Failed to update status for InferenceService "error-404-isvc-64e78": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-64e78": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-20T10:21:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:21:56Z" name: error-404-isvc-64e78.18c3f853e6616e61 namespace: kserve-ci-e2e-test resourceVersion: "21958" uid: 3609a371-d6cf-4098-8966-ab777b0a6189 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-64e78 namespace: kserve-ci-e2e-test resourceVersion: "21937" uid: f2492e4c-8884-4f45-806c-89953c3d796d kind: Event lastTimestamp: "2026-07-20T10:21:56Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-64e78": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-20T10:21:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:21:56Z" name: error-404-isvc-64e78.18c3f853e661a742 namespace: kserve-ci-e2e-test resourceVersion: "21959" uid: 41670122-f673-43a9-9dfd-f68864299b83 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:22:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-64e78 namespace: kserve-ci-e2e-test resourceVersion: "22442" uid: f2492e4c-8884-4f45-806c-89953c3d796d kind: Event lastTimestamp: "2026-07-20T10:22:44Z" message: InferenceService [error-404-isvc-64e78] is Ready metadata: creationTimestamp: "2026-07-20T10:22:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:22:44Z" name: error-404-isvc-64e78.18c3f85efeb31405 namespace: kserve-ci-e2e-test resourceVersion: "22443" uid: 1813dd20-b4fd-4b03-aeff-7fceace8e5a2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-20T10:31:22.166451Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6e423-predictor-5dbc84fc58-rktlq namespace: kserve-ci-e2e-test resourceVersion: "26849" uid: 9b0f6d24-13a4-475c-88de-e8b0a6e9767b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-6e423-predictor-5dbc84fc58-rktlq to ip-10-0-143-155.ec2.internal metadata: creationTimestamp: "2026-07-20T10:31:22Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-20T10:31:22Z" name: error-404-isvc-6e423-predictor-5dbc84fc58-rktlq.18c3f8d794c53ab2 namespace: kserve-ci-e2e-test resourceVersion: "26853" uid: 3cd96bbd-11b0-4289-9394-75fa53d46bef reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5dc669d9-69hkz source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:22Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6e423-predictor-5dbc84fc58-rktlq namespace: kserve-ci-e2e-test resourceVersion: "26851" uid: 9b0f6d24-13a4-475c-88de-e8b0a6e9767b kind: Event lastTimestamp: "2026-07-20T10:31:22Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-6e423-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-20T10:31:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:31:22Z" name: error-404-isvc-6e423-predictor-5dbc84fc58-rktlq.18c3f8d7a1662670 namespace: kserve-ci-e2e-test resourceVersion: "26869" uid: d6d4f927-b5b4-4935-bd69-cbe78fd7cd16 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:23Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6e423-predictor-5dbc84fc58-rktlq namespace: kserve-ci-e2e-test resourceVersion: "26859" uid: 9b0f6d24-13a4-475c-88de-e8b0a6e9767b kind: Event lastTimestamp: "2026-07-20T10:31:23Z" message: Add eth0 [10.133.0.55/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-20T10:31:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-20T10:31:23Z" name: error-404-isvc-6e423-predictor-5dbc84fc58-rktlq.18c3f8d7d2ff1ae4 namespace: kserve-ci-e2e-test resourceVersion: "26883" uid: 0802e221-3891-4562-80ac-877e56f15a2e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6e423-predictor-5dbc84fc58-rktlq namespace: kserve-ci-e2e-test resourceVersion: "26851" uid: 9b0f6d24-13a4-475c-88de-e8b0a6e9767b kind: Event lastTimestamp: "2026-07-20T10:31:23Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1771" already present on machine metadata: creationTimestamp: "2026-07-20T10:31:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:31:23Z" name: error-404-isvc-6e423-predictor-5dbc84fc58-rktlq.18c3f8d7d4ee9df0 namespace: kserve-ci-e2e-test resourceVersion: "26886" uid: 9bd34dd3-ea97-4fb7-93e0-c8b4a4b5426a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6e423-predictor-5dbc84fc58-rktlq namespace: kserve-ci-e2e-test resourceVersion: "26851" uid: 9b0f6d24-13a4-475c-88de-e8b0a6e9767b kind: Event lastTimestamp: "2026-07-20T10:31:23Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-20T10:31:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:31:23Z" name: error-404-isvc-6e423-predictor-5dbc84fc58-rktlq.18c3f8d7d7d7f573 namespace: kserve-ci-e2e-test resourceVersion: "26889" uid: 43f35c81-97d2-4858-9cf3-dacf358cda0c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6e423-predictor-5dbc84fc58-rktlq namespace: kserve-ci-e2e-test resourceVersion: "26851" uid: 9b0f6d24-13a4-475c-88de-e8b0a6e9767b kind: Event lastTimestamp: "2026-07-20T10:31:23Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-20T10:31:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:31:23Z" name: error-404-isvc-6e423-predictor-5dbc84fc58-rktlq.18c3f8d7d841ca2c namespace: kserve-ci-e2e-test resourceVersion: "26890" uid: e049d999-611f-4257-b8b5-1cd6b5082d47 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6e423-predictor-5dbc84fc58-rktlq namespace: kserve-ci-e2e-test resourceVersion: "26851" uid: 9b0f6d24-13a4-475c-88de-e8b0a6e9767b kind: Event lastTimestamp: "2026-07-20T10:31:23Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-20T10:31:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:31:23Z" name: error-404-isvc-6e423-predictor-5dbc84fc58-rktlq.18c3f8d7d84b750d namespace: kserve-ci-e2e-test resourceVersion: "26891" uid: 95f2916a-1926-49f6-b17a-fc36f6715dca reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6e423-predictor-5dbc84fc58-rktlq namespace: kserve-ci-e2e-test resourceVersion: "26851" uid: 9b0f6d24-13a4-475c-88de-e8b0a6e9767b kind: Event lastTimestamp: "2026-07-20T10:31:23Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-20T10:31:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:31:23Z" name: error-404-isvc-6e423-predictor-5dbc84fc58-rktlq.18c3f8d7dc8b3684 namespace: kserve-ci-e2e-test resourceVersion: "26892" uid: f80ec537-fbd7-4bf8-be2a-63414a1af79d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6e423-predictor-5dbc84fc58-rktlq namespace: kserve-ci-e2e-test resourceVersion: "26851" uid: 9b0f6d24-13a4-475c-88de-e8b0a6e9767b kind: Event lastTimestamp: "2026-07-20T10:31:23Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:31:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:31:23Z" name: error-404-isvc-6e423-predictor-5dbc84fc58-rktlq.18c3f8d7dcff451b namespace: kserve-ci-e2e-test resourceVersion: "26893" uid: d157470c-7954-4ee5-ac7c-8a2e47a94834 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-20T10:31:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6e423-predictor-5dbc84fc58-rktlq namespace: kserve-ci-e2e-test resourceVersion: "26851" uid: 9b0f6d24-13a4-475c-88de-e8b0a6e9767b kind: Event lastTimestamp: "2026-07-20T10:40:49Z" message: 'Readiness probe failed: dial tcp 10.133.0.55:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-20T10:31:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:40:49Z" name: error-404-isvc-6e423-predictor-5dbc84fc58-rktlq.18c3f8d7fbd75ec1 namespace: kserve-ci-e2e-test resourceVersion: "31080" uid: fec40591-f362-4e38-b71e-a9b13e90c4f9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:40:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6e423-predictor-5dbc84fc58-rktlq namespace: kserve-ci-e2e-test resourceVersion: "26851" uid: 9b0f6d24-13a4-475c-88de-e8b0a6e9767b kind: Event lastTimestamp: "2026-07-20T10:40:46Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-20T10:40:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:40:46Z" name: error-404-isvc-6e423-predictor-5dbc84fc58-rktlq.18c3f95b13e8bcd5 namespace: kserve-ci-e2e-test resourceVersion: "31056" uid: 1382fc06-0cd2-42dc-91c8-8ae157dfd2c9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:40:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6e423-predictor-5dbc84fc58-rktlq namespace: kserve-ci-e2e-test resourceVersion: "26851" uid: 9b0f6d24-13a4-475c-88de-e8b0a6e9767b kind: Event lastTimestamp: "2026-07-20T10:40:46Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:40:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:40:46Z" name: error-404-isvc-6e423-predictor-5dbc84fc58-rktlq.18c3f95b13e955a4 namespace: kserve-ci-e2e-test resourceVersion: "31058" uid: d6eaccb9-d258-424a-b1c3-60ab67ca17a0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:40:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6e423-predictor-5dbc84fc58-rktlq namespace: kserve-ci-e2e-test resourceVersion: "26851" uid: 9b0f6d24-13a4-475c-88de-e8b0a6e9767b kind: Event lastTimestamp: "2026-07-20T10:40:49Z" message: 'Readiness probe failed: Get "https://10.133.0.55:8643/healthz": dial tcp 10.133.0.55:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-20T10:40:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:40:49Z" name: error-404-isvc-6e423-predictor-5dbc84fc58-rktlq.18c3f95bc45850f7 namespace: kserve-ci-e2e-test resourceVersion: "31077" uid: e0eba04f-c55f-4431-afda-a09966a8e859 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:22Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-6e423-predictor-5dbc84fc58 namespace: kserve-ci-e2e-test resourceVersion: "26840" uid: 92f6a175-6d91-44ba-9977-30e2bdbe93cc kind: Event lastTimestamp: "2026-07-20T10:31:22Z" message: 'Created pod: error-404-isvc-6e423-predictor-5dbc84fc58-rktlq' metadata: creationTimestamp: "2026-07-20T10:31:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:31:22Z" name: error-404-isvc-6e423-predictor-5dbc84fc58.18c3f8d7942cae20 namespace: kserve-ci-e2e-test resourceVersion: "26850" uid: 1ac1c5fc-2d55-4500-ac82-65eaf3851774 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:22Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-6e423-predictor namespace: kserve-ci-e2e-test resourceVersion: "26838" uid: e059aacc-1366-4400-a2d1-62df606f2421 kind: Event lastTimestamp: "2026-07-20T10:31:22Z" message: Scaled up replica set error-404-isvc-6e423-predictor-5dbc84fc58 from 0 to 1 metadata: creationTimestamp: "2026-07-20T10:31:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:31:22Z" name: error-404-isvc-6e423-predictor.18c3f8d7924c45ab namespace: kserve-ci-e2e-test resourceVersion: "26842" uid: 3e979515-e370-4c44-8b36-ae77bffd4a16 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-20T10:31:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6e423-predictor namespace: kserve-ci-e2e-test resourceVersion: "26839" uid: b4a6df69-ed2a-4567-b26c-103f52c4f169 kind: Event lastTimestamp: "2026-07-20T10:31:52Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:31:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:31:52Z" name: error-404-isvc-6e423-predictor.18c3f8db10bb7d2d namespace: kserve-ci-e2e-test resourceVersion: "27160" uid: 4d690a87-51b2-4a52-afe7-0761265fefbf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-20T10:31:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6e423-predictor namespace: kserve-ci-e2e-test resourceVersion: "26839" uid: b4a6df69-ed2a-4567-b26c-103f52c4f169 kind: Event lastTimestamp: "2026-07-20T10:31:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:31:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:31:52Z" name: error-404-isvc-6e423-predictor.18c3f8db1104da16 namespace: kserve-ci-e2e-test resourceVersion: "27161" uid: e30f5061-6d9b-4dc3-bccf-2f85f7b94332 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-20T10:32:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6e423-predictor namespace: kserve-ci-e2e-test resourceVersion: "27032" uid: b4a6df69-ed2a-4567-b26c-103f52c4f169 kind: Event lastTimestamp: "2026-07-20T10:32:22Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:32:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:32:22Z" name: error-404-isvc-6e423-predictor.18c3f8e20e688b26 namespace: kserve-ci-e2e-test resourceVersion: "27460" uid: 859589d1-a9bc-4b75-83ea-54bbdd0f1ce0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-20T10:32:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6e423-predictor namespace: kserve-ci-e2e-test resourceVersion: "27032" uid: b4a6df69-ed2a-4567-b26c-103f52c4f169 kind: Event lastTimestamp: "2026-07-20T10:32:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:32:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:32:22Z" name: error-404-isvc-6e423-predictor.18c3f8e20edd2843 namespace: kserve-ci-e2e-test resourceVersion: "27461" uid: 1cd03d3b-d3aa-4886-99aa-827ac2b35bd8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-20T10:31:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6e423 namespace: kserve-ci-e2e-test resourceVersion: "26822" uid: ff2e984b-1c29-41d2-a593-0fed2b504f70 kind: Event lastTimestamp: "2026-07-20T10:32:10Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-20T10:31:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:32:10Z" name: error-404-isvc-6e423.18c3f8d7924c4fc0 namespace: kserve-ci-e2e-test resourceVersion: "27323" uid: 8fa79bc2-19f8-4d74-b5ea-6b22d7a20062 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6e423 namespace: kserve-ci-e2e-test resourceVersion: "26822" uid: ff2e984b-1c29-41d2-a593-0fed2b504f70 kind: Event lastTimestamp: "2026-07-20T10:31:22Z" message: 'Failed to update status for InferenceService "error-404-isvc-6e423": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-6e423": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-20T10:31:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:31:22Z" name: error-404-isvc-6e423.18c3f8d792e0ccf0 namespace: kserve-ci-e2e-test resourceVersion: "26845" uid: 6655c324-2fdf-4000-ad66-b1de0d158517 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6e423 namespace: kserve-ci-e2e-test resourceVersion: "26822" uid: ff2e984b-1c29-41d2-a593-0fed2b504f70 kind: Event lastTimestamp: "2026-07-20T10:31:22Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-6e423": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-20T10:31:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:31:22Z" name: error-404-isvc-6e423.18c3f8d792e1194c namespace: kserve-ci-e2e-test resourceVersion: "26846" uid: 278b0e96-0ea7-44ee-bdce-e730ea5ae370 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:32:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6e423 namespace: kserve-ci-e2e-test resourceVersion: "27320" uid: ff2e984b-1c29-41d2-a593-0fed2b504f70 kind: Event lastTimestamp: "2026-07-20T10:32:09Z" message: InferenceService [error-404-isvc-6e423] is Ready metadata: creationTimestamp: "2026-07-20T10:32:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:32:10Z" name: error-404-isvc-6e423.18c3f8e2b7da5a14 namespace: kserve-ci-e2e-test resourceVersion: "27321" uid: f2980ae0-f123-4254-ba9a-dc7a26a9f283 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-20T10:31:11.472839Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-78ab0-predictor-5d9cbd79b6-p5wz9 namespace: kserve-ci-e2e-test resourceVersion: "26575" uid: 45de7aec-00a2-4dd7-b579-9c7b6b655c67 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-78ab0-predictor-5d9cbd79b6-p5wz9 to ip-10-0-143-155.ec2.internal metadata: creationTimestamp: "2026-07-20T10:31:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-20T10:31:11Z" name: error-404-isvc-78ab0-predictor-5d9cbd79b6-p5wz9.18c3f8d51761b258 namespace: kserve-ci-e2e-test resourceVersion: "26579" uid: 98892b06-85e1-412c-b5bd-ff368a7dc19f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5dc669d9-69hkz source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:11Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-78ab0-predictor-5d9cbd79b6-p5wz9 namespace: kserve-ci-e2e-test resourceVersion: "26577" uid: 45de7aec-00a2-4dd7-b579-9c7b6b655c67 kind: Event lastTimestamp: "2026-07-20T10:31:11Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-78ab0-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-20T10:31:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:31:11Z" name: error-404-isvc-78ab0-predictor-5d9cbd79b6-p5wz9.18c3f8d523920afa namespace: kserve-ci-e2e-test resourceVersion: "26594" uid: 4dd688c4-f395-4d18-80a6-0805871f4c8d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:12Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-78ab0-predictor-5d9cbd79b6-p5wz9 namespace: kserve-ci-e2e-test resourceVersion: "26584" uid: 45de7aec-00a2-4dd7-b579-9c7b6b655c67 kind: Event lastTimestamp: "2026-07-20T10:31:12Z" message: Add eth0 [10.133.0.53/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-20T10:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-20T10:31:12Z" name: error-404-isvc-78ab0-predictor-5d9cbd79b6-p5wz9.18c3f8d554afe7b6 namespace: kserve-ci-e2e-test resourceVersion: "26647" uid: 97f3062b-56ad-4139-b43c-acc68b728691 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-78ab0-predictor-5d9cbd79b6-p5wz9 namespace: kserve-ci-e2e-test resourceVersion: "26577" uid: 45de7aec-00a2-4dd7-b579-9c7b6b655c67 kind: Event lastTimestamp: "2026-07-20T10:31:12Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1771" already present on machine metadata: creationTimestamp: "2026-07-20T10:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:31:12Z" name: error-404-isvc-78ab0-predictor-5d9cbd79b6-p5wz9.18c3f8d5560db7b7 namespace: kserve-ci-e2e-test resourceVersion: "26649" uid: af281f24-14e0-4acb-bec1-bbf8ed4cd049 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-78ab0-predictor-5d9cbd79b6-p5wz9 namespace: kserve-ci-e2e-test resourceVersion: "26577" uid: 45de7aec-00a2-4dd7-b579-9c7b6b655c67 kind: Event lastTimestamp: "2026-07-20T10:31:12Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-20T10:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:31:12Z" name: error-404-isvc-78ab0-predictor-5d9cbd79b6-p5wz9.18c3f8d5595bcb54 namespace: kserve-ci-e2e-test resourceVersion: "26650" uid: 39f94fa7-4e8d-4c82-917b-dd4b9184fb50 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-78ab0-predictor-5d9cbd79b6-p5wz9 namespace: kserve-ci-e2e-test resourceVersion: "26577" uid: 45de7aec-00a2-4dd7-b579-9c7b6b655c67 kind: Event lastTimestamp: "2026-07-20T10:31:12Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-20T10:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:31:12Z" name: error-404-isvc-78ab0-predictor-5d9cbd79b6-p5wz9.18c3f8d559cd61d6 namespace: kserve-ci-e2e-test resourceVersion: "26651" uid: 0499eda1-8682-44f8-bf85-e5af2a6a1f0f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-78ab0-predictor-5d9cbd79b6-p5wz9 namespace: kserve-ci-e2e-test resourceVersion: "26577" uid: 45de7aec-00a2-4dd7-b579-9c7b6b655c67 kind: Event lastTimestamp: "2026-07-20T10:31:12Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-20T10:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:31:12Z" name: error-404-isvc-78ab0-predictor-5d9cbd79b6-p5wz9.18c3f8d559d6a1b1 namespace: kserve-ci-e2e-test resourceVersion: "26653" uid: 8792f83d-2548-4d2a-823a-049da1b29930 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-78ab0-predictor-5d9cbd79b6-p5wz9 namespace: kserve-ci-e2e-test resourceVersion: "26577" uid: 45de7aec-00a2-4dd7-b579-9c7b6b655c67 kind: Event lastTimestamp: "2026-07-20T10:31:12Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-20T10:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:31:12Z" name: error-404-isvc-78ab0-predictor-5d9cbd79b6-p5wz9.18c3f8d55cd8f332 namespace: kserve-ci-e2e-test resourceVersion: "26654" uid: c1f3f625-83fb-4d37-b993-fdab1dd31a77 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-78ab0-predictor-5d9cbd79b6-p5wz9 namespace: kserve-ci-e2e-test resourceVersion: "26577" uid: 45de7aec-00a2-4dd7-b579-9c7b6b655c67 kind: Event lastTimestamp: "2026-07-20T10:31:12Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:31:12Z" name: error-404-isvc-78ab0-predictor-5d9cbd79b6-p5wz9.18c3f8d55d4335b0 namespace: kserve-ci-e2e-test resourceVersion: "26655" uid: 0571cd56-cb2d-4fc8-ac4a-10210b28a4bf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-20T10:31:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-78ab0-predictor-5d9cbd79b6-p5wz9 namespace: kserve-ci-e2e-test resourceVersion: "26577" uid: 45de7aec-00a2-4dd7-b579-9c7b6b655c67 kind: Event lastTimestamp: "2026-07-20T10:31:58Z" message: 'Readiness probe failed: dial tcp 10.133.0.53:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-20T10:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:31:58Z" name: error-404-isvc-78ab0-predictor-5d9cbd79b6-p5wz9.18c3f8d567fcb211 namespace: kserve-ci-e2e-test resourceVersion: "27210" uid: d637315e-7440-405a-989e-2e7f95b887d2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:48:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-78ab0-predictor-5d9cbd79b6-p5wz9 namespace: kserve-ci-e2e-test resourceVersion: "26577" uid: 45de7aec-00a2-4dd7-b579-9c7b6b655c67 kind: Event lastTimestamp: "2026-07-20T10:48:30Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-20T10:48:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:48:30Z" name: error-404-isvc-78ab0-predictor-5d9cbd79b6-p5wz9.18c3f9c7119d6f73 namespace: kserve-ci-e2e-test resourceVersion: "34284" uid: 1ba675f1-7620-4a94-8485-68267a7150fb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:48:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-78ab0-predictor-5d9cbd79b6-p5wz9 namespace: kserve-ci-e2e-test resourceVersion: "26577" uid: 45de7aec-00a2-4dd7-b579-9c7b6b655c67 kind: Event lastTimestamp: "2026-07-20T10:48:30Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:48:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:48:30Z" name: error-404-isvc-78ab0-predictor-5d9cbd79b6-p5wz9.18c3f9c7119dd16e namespace: kserve-ci-e2e-test resourceVersion: "34283" uid: 8c677b7f-cf03-4a02-ba6f-4c981b3b10eb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-78ab0-predictor-5d9cbd79b6 namespace: kserve-ci-e2e-test resourceVersion: "26567" uid: fd95b15b-2b4a-4f32-b30f-1c8cc1a3d89a kind: Event lastTimestamp: "2026-07-20T10:31:11Z" message: 'Created pod: error-404-isvc-78ab0-predictor-5d9cbd79b6-p5wz9' metadata: creationTimestamp: "2026-07-20T10:31:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:31:11Z" name: error-404-isvc-78ab0-predictor-5d9cbd79b6.18c3f8d51647d2a1 namespace: kserve-ci-e2e-test resourceVersion: "26576" uid: 9d26ae5a-9015-4171-addc-37c245798fd3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-78ab0-predictor namespace: kserve-ci-e2e-test resourceVersion: "26565" uid: 3f117a09-0ed6-4413-bbbd-06ccc850368f kind: Event lastTimestamp: "2026-07-20T10:31:11Z" message: Scaled up replica set error-404-isvc-78ab0-predictor-5d9cbd79b6 from 0 to 1 metadata: creationTimestamp: "2026-07-20T10:31:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:31:11Z" name: error-404-isvc-78ab0-predictor.18c3f8d5134d6cd5 namespace: kserve-ci-e2e-test resourceVersion: "26569" uid: 490228ed-2acd-4433-8d37-c8fdb3d99512 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-78ab0-predictor namespace: kserve-ci-e2e-test resourceVersion: "26566" uid: ee8bc22c-961a-4e69-ae34-76b3e977524e kind: Event lastTimestamp: "2026-07-20T10:31:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:31:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:31:26Z" name: error-404-isvc-78ab0-predictor.18c3f8d89135ce63 namespace: kserve-ci-e2e-test resourceVersion: "26941" uid: 56d77fd8-0e89-4aaf-b0e7-aacea80dac50 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-78ab0-predictor namespace: kserve-ci-e2e-test resourceVersion: "26566" uid: ee8bc22c-961a-4e69-ae34-76b3e977524e kind: Event lastTimestamp: "2026-07-20T10:31:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:31:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:31:26Z" name: error-404-isvc-78ab0-predictor.18c3f8d8919a2edf namespace: kserve-ci-e2e-test resourceVersion: "26943" uid: ba91b86a-4755-48a6-9cb2-e40d13c7c151 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-20T10:31:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-78ab0-predictor namespace: kserve-ci-e2e-test resourceVersion: "26942" uid: ee8bc22c-961a-4e69-ae34-76b3e977524e kind: Event lastTimestamp: "2026-07-20T10:32:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:31:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:32:41Z" name: error-404-isvc-78ab0-predictor.18c3f8dc1084cf83 namespace: kserve-ci-e2e-test resourceVersion: "27647" uid: 8eb43542-ade0-4050-b307-d4f67a2bd3d7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-20T10:31:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-78ab0-predictor namespace: kserve-ci-e2e-test resourceVersion: "26942" uid: ee8bc22c-961a-4e69-ae34-76b3e977524e kind: Event lastTimestamp: "2026-07-20T10:32:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:31:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:32:41Z" name: error-404-isvc-78ab0-predictor.18c3f8dc10ca8dc8 namespace: kserve-ci-e2e-test resourceVersion: "27648" uid: 75caf31d-9cd9-48c7-a15f-cbfb9f573fd9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-20T10:31:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-78ab0 namespace: kserve-ci-e2e-test resourceVersion: "26557" uid: df841845-2f2c-4fa4-a673-65e3173e1e6b kind: Event lastTimestamp: "2026-07-20T10:32:08Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-20T10:31:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:32:08Z" name: error-404-isvc-78ab0.18c3f8d51332bc65 namespace: kserve-ci-e2e-test resourceVersion: "27308" uid: 25208995-c386-4e31-905d-1af7ea8f3693 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-78ab0 namespace: kserve-ci-e2e-test resourceVersion: "26557" uid: df841845-2f2c-4fa4-a673-65e3173e1e6b kind: Event lastTimestamp: "2026-07-20T10:31:11Z" message: 'Failed to update status for InferenceService "error-404-isvc-78ab0": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-78ab0": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-20T10:31:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:31:11Z" name: error-404-isvc-78ab0.18c3f8d513f18411 namespace: kserve-ci-e2e-test resourceVersion: "26572" uid: 9b19722a-bd94-4138-8c0a-48ec1e40967e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-78ab0 namespace: kserve-ci-e2e-test resourceVersion: "26557" uid: df841845-2f2c-4fa4-a673-65e3173e1e6b kind: Event lastTimestamp: "2026-07-20T10:31:11Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-78ab0": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-20T10:31:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:31:11Z" name: error-404-isvc-78ab0.18c3f8d513f1ca73 namespace: kserve-ci-e2e-test resourceVersion: "26573" uid: 2a17b6ee-76e8-43c0-b59e-6147b46c1f1a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:32:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-78ab0 namespace: kserve-ci-e2e-test resourceVersion: "27306" uid: df841845-2f2c-4fa4-a673-65e3173e1e6b kind: Event lastTimestamp: "2026-07-20T10:32:08Z" message: InferenceService [error-404-isvc-78ab0] is Ready metadata: creationTimestamp: "2026-07-20T10:32:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:32:08Z" name: error-404-isvc-78ab0.18c3f8e27741d142 namespace: kserve-ci-e2e-test resourceVersion: "27307" uid: 7d1c88ff-1c9f-49d7-b656-95a08f9ab486 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-20T10:11:21.554220Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-78f99-predictor-868689f4b4-fhg95 namespace: kserve-ci-e2e-test resourceVersion: "16601" uid: 6ad7287e-93f2-4ea2-baa7-3fb58160b873 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-78f99-predictor-868689f4b4-fhg95 to ip-10-0-143-155.ec2.internal metadata: creationTimestamp: "2026-07-20T10:11:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-20T10:11:21Z" name: error-404-isvc-78f99-predictor-868689f4b4-fhg95.18c3f7c00ab47fc5 namespace: kserve-ci-e2e-test resourceVersion: "16614" uid: 1f096e1b-229f-44e4-a367-47080e806d66 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5dc669d9-69hkz source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:21Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-78f99-predictor-868689f4b4-fhg95 namespace: kserve-ci-e2e-test resourceVersion: "16623" uid: 6ad7287e-93f2-4ea2-baa7-3fb58160b873 kind: Event lastTimestamp: "2026-07-20T10:11:21Z" message: Add eth0 [10.133.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-20T10:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-20T10:11:21Z" name: error-404-isvc-78f99-predictor-868689f4b4-fhg95.18c3f7c02471922c namespace: kserve-ci-e2e-test resourceVersion: "16637" uid: 90452151-42a8-4097-bcd8-1417abea5a3a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-78f99-predictor-868689f4b4-fhg95 namespace: kserve-ci-e2e-test resourceVersion: "16611" uid: 6ad7287e-93f2-4ea2-baa7-3fb58160b873 kind: Event lastTimestamp: "2026-07-20T10:11:22Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1771" already present on machine metadata: creationTimestamp: "2026-07-20T10:11:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:11:22Z" name: error-404-isvc-78f99-predictor-868689f4b4-fhg95.18c3f7c025c15a89 namespace: kserve-ci-e2e-test resourceVersion: "16639" uid: 4b230deb-194b-47a4-bc51-a3fb72093785 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-78f99-predictor-868689f4b4-fhg95 namespace: kserve-ci-e2e-test resourceVersion: "16611" uid: 6ad7287e-93f2-4ea2-baa7-3fb58160b873 kind: Event lastTimestamp: "2026-07-20T10:11:22Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-20T10:11:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:11:22Z" name: error-404-isvc-78f99-predictor-868689f4b4-fhg95.18c3f7c0287ca33c namespace: kserve-ci-e2e-test resourceVersion: "16641" uid: 79bb9829-c6fc-4cd9-8e32-864a2626d37d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-78f99-predictor-868689f4b4-fhg95 namespace: kserve-ci-e2e-test resourceVersion: "16611" uid: 6ad7287e-93f2-4ea2-baa7-3fb58160b873 kind: Event lastTimestamp: "2026-07-20T10:11:22Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-20T10:11:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:11:22Z" name: error-404-isvc-78f99-predictor-868689f4b4-fhg95.18c3f7c028e22bd0 namespace: kserve-ci-e2e-test resourceVersion: "16642" uid: 5abf9acc-7960-496e-b83d-3c96f2a68835 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-78f99-predictor-868689f4b4-fhg95 namespace: kserve-ci-e2e-test resourceVersion: "16611" uid: 6ad7287e-93f2-4ea2-baa7-3fb58160b873 kind: Event lastTimestamp: "2026-07-20T10:11:22Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-20T10:11:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:11:22Z" name: error-404-isvc-78f99-predictor-868689f4b4-fhg95.18c3f7c028ebbd4c namespace: kserve-ci-e2e-test resourceVersion: "16643" uid: 920050f2-be6b-4a38-9f5e-90df32059300 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-78f99-predictor-868689f4b4-fhg95 namespace: kserve-ci-e2e-test resourceVersion: "16611" uid: 6ad7287e-93f2-4ea2-baa7-3fb58160b873 kind: Event lastTimestamp: "2026-07-20T10:11:22Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-20T10:11:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:11:22Z" name: error-404-isvc-78f99-predictor-868689f4b4-fhg95.18c3f7c02b8f625f namespace: kserve-ci-e2e-test resourceVersion: "16644" uid: d7408ebb-4066-4876-b8e7-588b57ab93a7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-78f99-predictor-868689f4b4-fhg95 namespace: kserve-ci-e2e-test resourceVersion: "16611" uid: 6ad7287e-93f2-4ea2-baa7-3fb58160b873 kind: Event lastTimestamp: "2026-07-20T10:11:22Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:11:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:11:22Z" name: error-404-isvc-78f99-predictor-868689f4b4-fhg95.18c3f7c02beda8af namespace: kserve-ci-e2e-test resourceVersion: "16645" uid: 5a198b05-d970-4378-81c4-10ffe285ad5d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-20T10:11:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-78f99-predictor-868689f4b4-fhg95 namespace: kserve-ci-e2e-test resourceVersion: "16611" uid: 6ad7287e-93f2-4ea2-baa7-3fb58160b873 kind: Event lastTimestamp: "2026-07-20T10:20:39Z" message: 'Readiness probe failed: dial tcp 10.133.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-20T10:11:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:20:39Z" name: error-404-isvc-78f99-predictor-868689f4b4-fhg95.18c3f7c0613d8a32 namespace: kserve-ci-e2e-test resourceVersion: "20980" uid: ad8d1619-933c-4d7a-8bef-4bc1d3218ad1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-78f99-predictor-868689f4b4-fhg95 namespace: kserve-ci-e2e-test resourceVersion: "16611" uid: 6ad7287e-93f2-4ea2-baa7-3fb58160b873 kind: Event lastTimestamp: "2026-07-20T10:20:36Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-20T10:20:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:20:36Z" name: error-404-isvc-78f99-predictor-868689f4b4-fhg95.18c3f8412cee5307 namespace: kserve-ci-e2e-test resourceVersion: "20874" uid: 2260b77b-6e1c-4120-b06b-fd59beeb90d7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-78f99-predictor-868689f4b4-fhg95 namespace: kserve-ci-e2e-test resourceVersion: "16611" uid: 6ad7287e-93f2-4ea2-baa7-3fb58160b873 kind: Event lastTimestamp: "2026-07-20T10:20:36Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:20:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:20:36Z" name: error-404-isvc-78f99-predictor-868689f4b4-fhg95.18c3f8412cf09bde namespace: kserve-ci-e2e-test resourceVersion: "20876" uid: 281977db-d51d-4778-b8e4-ef383f508975 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-78f99-predictor-868689f4b4-fhg95 namespace: kserve-ci-e2e-test resourceVersion: "16611" uid: 6ad7287e-93f2-4ea2-baa7-3fb58160b873 kind: Event lastTimestamp: "2026-07-20T10:20:39Z" message: 'Readiness probe failed: Get "https://10.133.0.41:8643/healthz": dial tcp 10.133.0.41:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-20T10:20:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:20:39Z" name: error-404-isvc-78f99-predictor-868689f4b4-fhg95.18c3f841d5cfd850 namespace: kserve-ci-e2e-test resourceVersion: "20978" uid: 712d3200-3d46-409f-9127-c66125b7557e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-78f99-predictor-868689f4b4 namespace: kserve-ci-e2e-test resourceVersion: "16588" uid: 2c32dc71-eb68-4e27-85aa-2ccdac4cafec kind: Event lastTimestamp: "2026-07-20T10:11:21Z" message: 'Created pod: error-404-isvc-78f99-predictor-868689f4b4-fhg95' metadata: creationTimestamp: "2026-07-20T10:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:11:21Z" name: error-404-isvc-78f99-predictor-868689f4b4.18c3f7c009f72b0a namespace: kserve-ci-e2e-test resourceVersion: "16609" uid: 8792013b-8e7e-4bba-900f-fad06d2b55f2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-78f99-predictor namespace: kserve-ci-e2e-test resourceVersion: "16586" uid: d5e75f0e-857c-493b-80bc-5925eeb35b9e kind: Event lastTimestamp: "2026-07-20T10:11:21Z" message: Scaled up replica set error-404-isvc-78f99-predictor-868689f4b4 from 0 to 1 metadata: creationTimestamp: "2026-07-20T10:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:11:21Z" name: error-404-isvc-78f99-predictor.18c3f7c00780f56f namespace: kserve-ci-e2e-test resourceVersion: "16589" uid: 306870fa-ff53-4fab-b55e-734f4f48174b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-78f99-predictor namespace: kserve-ci-e2e-test resourceVersion: "16587" uid: dcfd8508-0f4b-40c5-9a40-8a86ac461d68 kind: Event lastTimestamp: "2026-07-20T10:11:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:11:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:11:36Z" name: error-404-isvc-78f99-predictor.18c3f7c385f1c8b8 namespace: kserve-ci-e2e-test resourceVersion: "16826" uid: 943e4740-3e72-46aa-89c7-09b0107a6502 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-78f99-predictor namespace: kserve-ci-e2e-test resourceVersion: "16587" uid: dcfd8508-0f4b-40c5-9a40-8a86ac461d68 kind: Event lastTimestamp: "2026-07-20T10:11:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:11:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:11:36Z" name: error-404-isvc-78f99-predictor.18c3f7c38672e124 namespace: kserve-ci-e2e-test resourceVersion: "16827" uid: e1a4950e-f8a0-41eb-b011-af9f50cb0e35 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-20T10:11:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-78f99-predictor namespace: kserve-ci-e2e-test resourceVersion: "16825" uid: dcfd8508-0f4b-40c5-9a40-8a86ac461d68 kind: Event lastTimestamp: "2026-07-20T10:12:36Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:11:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:12:36Z" name: error-404-isvc-78f99-predictor.18c3f7c705080d81 namespace: kserve-ci-e2e-test resourceVersion: "17336" uid: 355c4ff6-d8b3-4e9b-9f29-dd71b55fda44 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-20T10:11:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-78f99-predictor namespace: kserve-ci-e2e-test resourceVersion: "16825" uid: dcfd8508-0f4b-40c5-9a40-8a86ac461d68 kind: Event lastTimestamp: "2026-07-20T10:12:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:11:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:12:36Z" name: error-404-isvc-78f99-predictor.18c3f7c7054eaca4 namespace: kserve-ci-e2e-test resourceVersion: "17337" uid: 54632579-a0d4-42a5-8e01-b1d03002c72d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-20T10:11:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-78f99 namespace: kserve-ci-e2e-test resourceVersion: "16560" uid: 16e95de6-3feb-4a27-9f1e-5a361207a0cd kind: Event lastTimestamp: "2026-07-20T10:12:09Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-20T10:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:12:09Z" name: error-404-isvc-78f99.18c3f7c007873893 namespace: kserve-ci-e2e-test resourceVersion: "17099" uid: 435df4bc-7d0d-4597-ad77-21681bfb72f8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:12:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-78f99 namespace: kserve-ci-e2e-test resourceVersion: "17093" uid: 16e95de6-3feb-4a27-9f1e-5a361207a0cd kind: Event lastTimestamp: "2026-07-20T10:12:09Z" message: InferenceService [error-404-isvc-78f99] is Ready metadata: creationTimestamp: "2026-07-20T10:12:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:12:09Z" name: error-404-isvc-78f99.18c3f7cb1cd50ded namespace: kserve-ci-e2e-test resourceVersion: "17094" uid: 7241ea04-1ecd-447e-9765-2606db363c66 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-20T10:10:31.500195Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7f819-predictor-78699b444d-jdwpj namespace: kserve-ci-e2e-test resourceVersion: "15970" uid: 5dabde92-fd33-4162-8e56-47e594b50e7a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-7f819-predictor-78699b444d-jdwpj to ip-10-0-143-155.ec2.internal metadata: creationTimestamp: "2026-07-20T10:10:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-20T10:10:31Z" name: error-404-isvc-7f819-predictor-78699b444d-jdwpj.18c3f7b46340a618 namespace: kserve-ci-e2e-test resourceVersion: "15976" uid: 124c9ff9-cfb5-4842-9f2d-3dbf6f3c2fef reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5dc669d9-69hkz source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:31Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7f819-predictor-78699b444d-jdwpj namespace: kserve-ci-e2e-test resourceVersion: "15974" uid: 5dabde92-fd33-4162-8e56-47e594b50e7a kind: Event lastTimestamp: "2026-07-20T10:10:31Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-7f819-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-20T10:10:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:10:31Z" name: error-404-isvc-7f819-predictor-78699b444d-jdwpj.18c3f7b472134878 namespace: kserve-ci-e2e-test resourceVersion: "15991" uid: a2c84043-f517-4d93-b60e-f8cc39eb8ecb reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:32Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7f819-predictor-78699b444d-jdwpj namespace: kserve-ci-e2e-test resourceVersion: "15981" uid: 5dabde92-fd33-4162-8e56-47e594b50e7a kind: Event lastTimestamp: "2026-07-20T10:10:32Z" message: Add eth0 [10.133.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-20T10:10:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-20T10:10:32Z" name: error-404-isvc-7f819-predictor-78699b444d-jdwpj.18c3f7b4a10126a4 namespace: kserve-ci-e2e-test resourceVersion: "16008" uid: 7e748152-4cae-44de-be1c-a1dae161fa87 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7f819-predictor-78699b444d-jdwpj namespace: kserve-ci-e2e-test resourceVersion: "15974" uid: 5dabde92-fd33-4162-8e56-47e594b50e7a kind: Event lastTimestamp: "2026-07-20T10:10:32Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1771" already present on machine metadata: creationTimestamp: "2026-07-20T10:10:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:10:32Z" name: error-404-isvc-7f819-predictor-78699b444d-jdwpj.18c3f7b4a2e9d0b1 namespace: kserve-ci-e2e-test resourceVersion: "16011" uid: 0ef0a69a-5790-451b-8856-0d64371e3d2c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7f819-predictor-78699b444d-jdwpj namespace: kserve-ci-e2e-test resourceVersion: "15974" uid: 5dabde92-fd33-4162-8e56-47e594b50e7a kind: Event lastTimestamp: "2026-07-20T10:10:32Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-20T10:10:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:10:32Z" name: error-404-isvc-7f819-predictor-78699b444d-jdwpj.18c3f7b4a5b9fdf4 namespace: kserve-ci-e2e-test resourceVersion: "16012" uid: 4f7df167-fe12-43dc-8bfe-6861bcf0e612 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7f819-predictor-78699b444d-jdwpj namespace: kserve-ci-e2e-test resourceVersion: "15974" uid: 5dabde92-fd33-4162-8e56-47e594b50e7a kind: Event lastTimestamp: "2026-07-20T10:10:32Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-20T10:10:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:10:32Z" name: error-404-isvc-7f819-predictor-78699b444d-jdwpj.18c3f7b4a61735f9 namespace: kserve-ci-e2e-test resourceVersion: "16013" uid: 7a435132-4843-4947-9fce-f6ed9edd155c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7f819-predictor-78699b444d-jdwpj namespace: kserve-ci-e2e-test resourceVersion: "15974" uid: 5dabde92-fd33-4162-8e56-47e594b50e7a kind: Event lastTimestamp: "2026-07-20T10:10:32Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-20T10:10:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:10:32Z" name: error-404-isvc-7f819-predictor-78699b444d-jdwpj.18c3f7b4a6200e0c namespace: kserve-ci-e2e-test resourceVersion: "16014" uid: 50d6b29e-fbc5-4601-bc33-0555e76a3dfe reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7f819-predictor-78699b444d-jdwpj namespace: kserve-ci-e2e-test resourceVersion: "15974" uid: 5dabde92-fd33-4162-8e56-47e594b50e7a kind: Event lastTimestamp: "2026-07-20T10:10:32Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-20T10:10:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:10:32Z" name: error-404-isvc-7f819-predictor-78699b444d-jdwpj.18c3f7b4aa1539f3 namespace: kserve-ci-e2e-test resourceVersion: "16015" uid: 7377eee2-6b76-49f3-a4f9-3f172fc0527b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7f819-predictor-78699b444d-jdwpj namespace: kserve-ci-e2e-test resourceVersion: "15974" uid: 5dabde92-fd33-4162-8e56-47e594b50e7a kind: Event lastTimestamp: "2026-07-20T10:10:32Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:10:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:10:32Z" name: error-404-isvc-7f819-predictor-78699b444d-jdwpj.18c3f7b4aa799296 namespace: kserve-ci-e2e-test resourceVersion: "16016" uid: 6c044eb7-d4f7-4719-bfac-c56cb7631d96 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-20T10:10:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7f819-predictor-78699b444d-jdwpj namespace: kserve-ci-e2e-test resourceVersion: "15974" uid: 5dabde92-fd33-4162-8e56-47e594b50e7a kind: Event lastTimestamp: "2026-07-20T10:19:49Z" message: 'Readiness probe failed: dial tcp 10.133.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-20T10:10:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:19:49Z" name: error-404-isvc-7f819-predictor-78699b444d-jdwpj.18c3f7b4ea3c7414 namespace: kserve-ci-e2e-test resourceVersion: "20447" uid: b05be6b9-12c0-43d0-a8b4-63624f1ec98a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:19:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7f819-predictor-78699b444d-jdwpj namespace: kserve-ci-e2e-test resourceVersion: "15974" uid: 5dabde92-fd33-4162-8e56-47e594b50e7a kind: Event lastTimestamp: "2026-07-20T10:19:46Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-20T10:19:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:19:46Z" name: error-404-isvc-7f819-predictor-78699b444d-jdwpj.18c3f835b14548c6 namespace: kserve-ci-e2e-test resourceVersion: "20392" uid: 8175545b-d8d2-4558-a494-2f3afeb82664 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:19:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7f819-predictor-78699b444d-jdwpj namespace: kserve-ci-e2e-test resourceVersion: "15974" uid: 5dabde92-fd33-4162-8e56-47e594b50e7a kind: Event lastTimestamp: "2026-07-20T10:19:46Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:19:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:19:46Z" name: error-404-isvc-7f819-predictor-78699b444d-jdwpj.18c3f835b145c343 namespace: kserve-ci-e2e-test resourceVersion: "20393" uid: 49aa0866-089f-41dc-b4dd-e0e6bf108dc8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:19:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7f819-predictor-78699b444d-jdwpj namespace: kserve-ci-e2e-test resourceVersion: "15974" uid: 5dabde92-fd33-4162-8e56-47e594b50e7a kind: Event lastTimestamp: "2026-07-20T10:19:49Z" message: 'Readiness probe failed: Get "https://10.133.0.39:8643/healthz": dial tcp 10.133.0.39:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-20T10:19:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:19:49Z" name: error-404-isvc-7f819-predictor-78699b444d-jdwpj.18c3f8365eca7184 namespace: kserve-ci-e2e-test resourceVersion: "20444" uid: 4292a0d8-ba99-4463-9306-226a6d5c72a1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-7f819-predictor-78699b444d namespace: kserve-ci-e2e-test resourceVersion: "15963" uid: 03633609-3002-42e4-8e5d-7e73df462152 kind: Event lastTimestamp: "2026-07-20T10:10:31Z" message: 'Created pod: error-404-isvc-7f819-predictor-78699b444d-jdwpj' metadata: creationTimestamp: "2026-07-20T10:10:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:10:31Z" name: error-404-isvc-7f819-predictor-78699b444d.18c3f7b462b67343 namespace: kserve-ci-e2e-test resourceVersion: "15972" uid: 3576abf1-0f26-4332-9e4a-5ba86b548ed6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-7f819-predictor namespace: kserve-ci-e2e-test resourceVersion: "15961" uid: 91d580b2-271d-42db-b42b-c42dd5501797 kind: Event lastTimestamp: "2026-07-20T10:10:31Z" message: Scaled up replica set error-404-isvc-7f819-predictor-78699b444d from 0 to 1 metadata: creationTimestamp: "2026-07-20T10:10:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:10:31Z" name: error-404-isvc-7f819-predictor.18c3f7b461342345 namespace: kserve-ci-e2e-test resourceVersion: "15964" uid: c4e15105-fad3-410c-b7ae-0c4b08a37847 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-20T10:10:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7f819-predictor namespace: kserve-ci-e2e-test resourceVersion: "15962" uid: 03fca740-22dd-4df4-bfcc-0865c0ba2ed4 kind: Event lastTimestamp: "2026-07-20T10:11:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:10:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:11:01Z" name: error-404-isvc-7f819-predictor.18c3f7b7e0a4e5f4 namespace: kserve-ci-e2e-test resourceVersion: "16283" uid: 115e22ca-f8dd-4c3c-a0c2-ef82951673a1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-20T10:10:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7f819-predictor namespace: kserve-ci-e2e-test resourceVersion: "15962" uid: 03fca740-22dd-4df4-bfcc-0865c0ba2ed4 kind: Event lastTimestamp: "2026-07-20T10:11:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:10:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:11:01Z" name: error-404-isvc-7f819-predictor.18c3f7b7e15f730b namespace: kserve-ci-e2e-test resourceVersion: "16284" uid: 64e90781-5eb0-4040-b95b-647eb04e3f39 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-20T10:11:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7f819-predictor namespace: kserve-ci-e2e-test resourceVersion: "16148" uid: 03fca740-22dd-4df4-bfcc-0865c0ba2ed4 kind: Event lastTimestamp: "2026-07-20T10:11:31Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:11:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:11:31Z" name: error-404-isvc-7f819-predictor.18c3f7bedf3c2b54 namespace: kserve-ci-e2e-test resourceVersion: "16766" uid: 9d0f7c88-88ea-4960-8d98-52f7525a2e20 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-20T10:11:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7f819-predictor namespace: kserve-ci-e2e-test resourceVersion: "16148" uid: 03fca740-22dd-4df4-bfcc-0865c0ba2ed4 kind: Event lastTimestamp: "2026-07-20T10:11:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:11:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:11:31Z" name: error-404-isvc-7f819-predictor.18c3f7bedfb597c8 namespace: kserve-ci-e2e-test resourceVersion: "16767" uid: 4a88532a-8c9d-4eba-b30e-5960091c4a4e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-20T10:10:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7f819 namespace: kserve-ci-e2e-test resourceVersion: "15945" uid: 9c6a03ea-8c0f-43ee-9d5a-a62f6be15f76 kind: Event lastTimestamp: "2026-07-20T10:11:19Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-20T10:10:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:11:19Z" name: error-404-isvc-7f819.18c3f7b46135a82e namespace: kserve-ci-e2e-test resourceVersion: "16491" uid: 1e9249cb-37f9-47a5-9ce9-df7790c14945 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7f819 namespace: kserve-ci-e2e-test resourceVersion: "15945" uid: 9c6a03ea-8c0f-43ee-9d5a-a62f6be15f76 kind: Event lastTimestamp: "2026-07-20T10:10:31Z" message: 'Failed to update status for InferenceService "error-404-isvc-7f819": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-7f819": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-20T10:10:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:10:31Z" name: error-404-isvc-7f819.18c3f7b461bbde16 namespace: kserve-ci-e2e-test resourceVersion: "15967" uid: 24bf2e54-0c33-41d6-955e-5d1307c497f3 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7f819 namespace: kserve-ci-e2e-test resourceVersion: "15945" uid: 9c6a03ea-8c0f-43ee-9d5a-a62f6be15f76 kind: Event lastTimestamp: "2026-07-20T10:10:31Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-7f819": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-20T10:10:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:10:31Z" name: error-404-isvc-7f819.18c3f7b461bc439f namespace: kserve-ci-e2e-test resourceVersion: "15968" uid: ad0ba053-15aa-4d21-b5f2-f5b968816dae reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7f819 namespace: kserve-ci-e2e-test resourceVersion: "16481" uid: 9c6a03ea-8c0f-43ee-9d5a-a62f6be15f76 kind: Event lastTimestamp: "2026-07-20T10:11:19Z" message: InferenceService [error-404-isvc-7f819] is Ready metadata: creationTimestamp: "2026-07-20T10:11:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:11:19Z" name: error-404-isvc-7f819.18c3f7bfa5f1a2e9 namespace: kserve-ci-e2e-test resourceVersion: "16482" uid: 18002abb-f52a-4b29-9ae3-18e34e224583 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-20T10:30:11.873428Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a956e-predictor-78896946fd-x7gsj namespace: kserve-ci-e2e-test resourceVersion: "25939" uid: 53ee06bf-5653-4a01-9147-c0adf85c66a5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-a956e-predictor-78896946fd-x7gsj to ip-10-0-143-155.ec2.internal metadata: creationTimestamp: "2026-07-20T10:30:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-20T10:30:11Z" name: error-404-isvc-a956e-predictor-78896946fd-x7gsj.18c3f8c736fac386 namespace: kserve-ci-e2e-test resourceVersion: "25945" uid: 4a945a07-ffe7-4ab7-8b1b-b751e2607086 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5dc669d9-69hkz source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:30:12Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a956e-predictor-78896946fd-x7gsj namespace: kserve-ci-e2e-test resourceVersion: "25955" uid: 53ee06bf-5653-4a01-9147-c0adf85c66a5 kind: Event lastTimestamp: "2026-07-20T10:30:12Z" message: Add eth0 [10.133.0.51/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-20T10:30:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-20T10:30:12Z" name: error-404-isvc-a956e-predictor-78896946fd-x7gsj.18c3f8c7516eda77 namespace: kserve-ci-e2e-test resourceVersion: "25985" uid: 43ca4393-d735-4647-9525-005dc0a450a5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:30:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a956e-predictor-78896946fd-x7gsj namespace: kserve-ci-e2e-test resourceVersion: "25942" uid: 53ee06bf-5653-4a01-9147-c0adf85c66a5 kind: Event lastTimestamp: "2026-07-20T10:30:12Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1771" already present on machine metadata: creationTimestamp: "2026-07-20T10:30:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:30:12Z" name: error-404-isvc-a956e-predictor-78896946fd-x7gsj.18c3f8c752d012a3 namespace: kserve-ci-e2e-test resourceVersion: "25987" uid: 9ff800af-0ce0-475b-91b6-cf382abbe7fd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:30:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a956e-predictor-78896946fd-x7gsj namespace: kserve-ci-e2e-test resourceVersion: "25942" uid: 53ee06bf-5653-4a01-9147-c0adf85c66a5 kind: Event lastTimestamp: "2026-07-20T10:30:12Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-20T10:30:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:30:12Z" name: error-404-isvc-a956e-predictor-78896946fd-x7gsj.18c3f8c755956da3 namespace: kserve-ci-e2e-test resourceVersion: "25990" uid: da689ab5-cb59-41ce-8d11-94f51bc6d184 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:30:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a956e-predictor-78896946fd-x7gsj namespace: kserve-ci-e2e-test resourceVersion: "25942" uid: 53ee06bf-5653-4a01-9147-c0adf85c66a5 kind: Event lastTimestamp: "2026-07-20T10:30:12Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-20T10:30:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:30:12Z" name: error-404-isvc-a956e-predictor-78896946fd-x7gsj.18c3f8c756037a64 namespace: kserve-ci-e2e-test resourceVersion: "25991" uid: b01877e0-8a7d-47bd-8262-9e5d0c7dc01d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:30:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a956e-predictor-78896946fd-x7gsj namespace: kserve-ci-e2e-test resourceVersion: "25942" uid: 53ee06bf-5653-4a01-9147-c0adf85c66a5 kind: Event lastTimestamp: "2026-07-20T10:30:12Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-20T10:30:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:30:12Z" name: error-404-isvc-a956e-predictor-78896946fd-x7gsj.18c3f8c7560ca5ef namespace: kserve-ci-e2e-test resourceVersion: "25993" uid: cc592036-f08b-49f7-8c31-b4c285165453 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:30:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a956e-predictor-78896946fd-x7gsj namespace: kserve-ci-e2e-test resourceVersion: "25942" uid: 53ee06bf-5653-4a01-9147-c0adf85c66a5 kind: Event lastTimestamp: "2026-07-20T10:30:12Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-20T10:30:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:30:12Z" name: error-404-isvc-a956e-predictor-78896946fd-x7gsj.18c3f8c759230b9a namespace: kserve-ci-e2e-test resourceVersion: "25995" uid: 7d8abe33-bd1b-4fe4-8dc8-76df5b05d691 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:30:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a956e-predictor-78896946fd-x7gsj namespace: kserve-ci-e2e-test resourceVersion: "25942" uid: 53ee06bf-5653-4a01-9147-c0adf85c66a5 kind: Event lastTimestamp: "2026-07-20T10:30:12Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:30:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:30:12Z" name: error-404-isvc-a956e-predictor-78896946fd-x7gsj.18c3f8c75992a199 namespace: kserve-ci-e2e-test resourceVersion: "25996" uid: e82a5a7d-96ec-450f-84d5-987a545a47b8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-20T10:30:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a956e-predictor-78896946fd-x7gsj namespace: kserve-ci-e2e-test resourceVersion: "25942" uid: 53ee06bf-5653-4a01-9147-c0adf85c66a5 kind: Event lastTimestamp: "2026-07-20T10:30:48Z" message: 'Readiness probe failed: dial tcp 10.133.0.51:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-20T10:30:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:30:48Z" name: error-404-isvc-a956e-predictor-78896946fd-x7gsj.18c3f8c75d687049 namespace: kserve-ci-e2e-test resourceVersion: "26300" uid: 8695c132-44f2-4dc0-bd61-5cecad0e4f5b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a956e-predictor-78896946fd-x7gsj namespace: kserve-ci-e2e-test resourceVersion: "25942" uid: 53ee06bf-5653-4a01-9147-c0adf85c66a5 kind: Event lastTimestamp: "2026-07-20T10:31:22Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-20T10:31:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:31:22Z" name: error-404-isvc-a956e-predictor-78896946fd-x7gsj.18c3f8d78d20ee8a namespace: kserve-ci-e2e-test resourceVersion: "26816" uid: 8f1d96b3-d92b-4fb4-951a-3fb8dac1c1bd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a956e-predictor-78896946fd-x7gsj namespace: kserve-ci-e2e-test resourceVersion: "25942" uid: 53ee06bf-5653-4a01-9147-c0adf85c66a5 kind: Event lastTimestamp: "2026-07-20T10:31:22Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:31:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:31:22Z" name: error-404-isvc-a956e-predictor-78896946fd-x7gsj.18c3f8d78d21b10a namespace: kserve-ci-e2e-test resourceVersion: "26818" uid: bbc4e88d-6c36-49e9-9d7b-45974352fba4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a956e-predictor-78896946fd-x7gsj namespace: kserve-ci-e2e-test resourceVersion: "25942" uid: 53ee06bf-5653-4a01-9147-c0adf85c66a5 kind: Event lastTimestamp: "2026-07-20T10:31:23Z" message: 'Readiness probe failed: Get "https://10.133.0.51:8643/healthz": dial tcp 10.133.0.51:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-20T10:31:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:31:23Z" name: error-404-isvc-a956e-predictor-78896946fd-x7gsj.18c3f8d7e5d2f0c6 namespace: kserve-ci-e2e-test resourceVersion: "26894" uid: f4e27e72-bf3a-4df3-9c5b-673697088216 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:30:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-a956e-predictor-78896946fd namespace: kserve-ci-e2e-test resourceVersion: "25926" uid: 67909d2b-1fe4-4faf-883b-d0dc7ecf8d75 kind: Event lastTimestamp: "2026-07-20T10:30:11Z" message: 'Created pod: error-404-isvc-a956e-predictor-78896946fd-x7gsj' metadata: creationTimestamp: "2026-07-20T10:30:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:30:11Z" name: error-404-isvc-a956e-predictor-78896946fd.18c3f8c73614037f namespace: kserve-ci-e2e-test resourceVersion: "25941" uid: 3977d0b8-07d6-4f8a-a460-29d679ef0056 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:30:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-a956e-predictor namespace: kserve-ci-e2e-test resourceVersion: "25915" uid: 9b468b09-6bbe-4a8e-8366-9a98ff34f48e kind: Event lastTimestamp: "2026-07-20T10:30:11Z" message: Scaled up replica set error-404-isvc-a956e-predictor-78896946fd from 0 to 1 metadata: creationTimestamp: "2026-07-20T10:30:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:30:11Z" name: error-404-isvc-a956e-predictor.18c3f8c732eb91cc namespace: kserve-ci-e2e-test resourceVersion: "25928" uid: 8f6f7357-ae8e-4cdc-9b3b-bbb4a9e7f2c7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-20T10:30:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a956e-predictor namespace: kserve-ci-e2e-test resourceVersion: "25924" uid: 8b3d2e22-49d9-4633-a1b5-03fb705f404b kind: Event lastTimestamp: "2026-07-20T10:30:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:30:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:30:56Z" name: error-404-isvc-a956e-predictor.18c3f8cab1562001 namespace: kserve-ci-e2e-test resourceVersion: "26359" uid: a0f884a2-cb10-4c0a-8b9c-763235c628a9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-20T10:30:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a956e-predictor namespace: kserve-ci-e2e-test resourceVersion: "25924" uid: 8b3d2e22-49d9-4633-a1b5-03fb705f404b kind: Event lastTimestamp: "2026-07-20T10:30:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:30:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:30:56Z" name: error-404-isvc-a956e-predictor.18c3f8cab1b97484 namespace: kserve-ci-e2e-test resourceVersion: "26360" uid: 46f61007-83e9-4205-9cc5-75204cf963f0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-20T10:30:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a956e-predictor namespace: kserve-ci-e2e-test resourceVersion: "26129" uid: 8b3d2e22-49d9-4633-a1b5-03fb705f404b kind: Event lastTimestamp: "2026-07-20T10:31:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:30:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:31:11Z" name: error-404-isvc-a956e-predictor.18c3f8ce30598331 namespace: kserve-ci-e2e-test resourceVersion: "26631" uid: 23a4d790-1edf-4316-8e8f-91a9cdb1e77e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-20T10:30:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a956e-predictor namespace: kserve-ci-e2e-test resourceVersion: "26129" uid: 8b3d2e22-49d9-4633-a1b5-03fb705f404b kind: Event lastTimestamp: "2026-07-20T10:31:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:30:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:31:11Z" name: error-404-isvc-a956e-predictor.18c3f8ce30c28547 namespace: kserve-ci-e2e-test resourceVersion: "26633" uid: b6bea2d9-59e5-4cd9-b69b-4ddd63b75d39 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-20T10:30:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a956e namespace: kserve-ci-e2e-test resourceVersion: "25899" uid: 9264b6e6-71c7-439b-99c6-85c9184599e8 kind: Event lastTimestamp: "2026-07-20T10:30:58Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-20T10:30:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:30:58Z" name: error-404-isvc-a956e.18c3f8c73355733f namespace: kserve-ci-e2e-test resourceVersion: "26379" uid: 86052a70-976f-4c88-aa4b-ba6fe6604845 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:30:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a956e namespace: kserve-ci-e2e-test resourceVersion: "25899" uid: 9264b6e6-71c7-439b-99c6-85c9184599e8 kind: Event lastTimestamp: "2026-07-20T10:30:11Z" message: 'Failed to update status for InferenceService "error-404-isvc-a956e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-a956e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-20T10:30:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:30:11Z" name: error-404-isvc-a956e.18c3f8c7341e498b namespace: kserve-ci-e2e-test resourceVersion: "25934" uid: aa43adce-3013-40ca-967c-515ec1dc3ee4 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:30:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a956e namespace: kserve-ci-e2e-test resourceVersion: "25899" uid: 9264b6e6-71c7-439b-99c6-85c9184599e8 kind: Event lastTimestamp: "2026-07-20T10:30:11Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-a956e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-20T10:30:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:30:11Z" name: error-404-isvc-a956e.18c3f8c7341e9559 namespace: kserve-ci-e2e-test resourceVersion: "25937" uid: 20321302-b39a-435b-ac21-8e738cfe228c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:30:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a956e namespace: kserve-ci-e2e-test resourceVersion: "26377" uid: 9264b6e6-71c7-439b-99c6-85c9184599e8 kind: Event lastTimestamp: "2026-07-20T10:30:58Z" message: InferenceService [error-404-isvc-a956e] is Ready metadata: creationTimestamp: "2026-07-20T10:30:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:30:58Z" name: error-404-isvc-a956e.18c3f8d21a0ca74b namespace: kserve-ci-e2e-test resourceVersion: "26378" uid: 6a8d8be2-a134-4345-b539-7ecc4a98a392 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-20T10:09:01.604267Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ab339-predictor-98847c874-24mfc namespace: kserve-ci-e2e-test resourceVersion: "14942" uid: 6633df2b-88a5-450c-875f-19dc0b7f0157 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-ab339-predictor-98847c874-24mfc to ip-10-0-143-155.ec2.internal metadata: creationTimestamp: "2026-07-20T10:09:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-20T10:09:01Z" name: error-404-isvc-ab339-predictor-98847c874-24mfc.18c3f79f75099790 namespace: kserve-ci-e2e-test resourceVersion: "14947" uid: 6a21a189-e805-4baa-9a3a-09965c14298a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5dc669d9-69hkz source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:01Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ab339-predictor-98847c874-24mfc namespace: kserve-ci-e2e-test resourceVersion: "14944" uid: 6633df2b-88a5-450c-875f-19dc0b7f0157 kind: Event lastTimestamp: "2026-07-20T10:09:01Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-ab339-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-20T10:09:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:09:01Z" name: error-404-isvc-ab339-predictor-98847c874-24mfc.18c3f79f832bb241 namespace: kserve-ci-e2e-test resourceVersion: "14985" uid: 8ee0571b-0959-4969-bc28-1920c2cfaa6b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:02Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ab339-predictor-98847c874-24mfc namespace: kserve-ci-e2e-test resourceVersion: "14952" uid: 6633df2b-88a5-450c-875f-19dc0b7f0157 kind: Event lastTimestamp: "2026-07-20T10:09:02Z" message: Add eth0 [10.133.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-20T10:09:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-20T10:09:02Z" name: error-404-isvc-ab339-predictor-98847c874-24mfc.18c3f79fb45c59f1 namespace: kserve-ci-e2e-test resourceVersion: "15023" uid: 94f196ef-1565-40da-883a-2c9aa4fcfa24 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ab339-predictor-98847c874-24mfc namespace: kserve-ci-e2e-test resourceVersion: "14944" uid: 6633df2b-88a5-450c-875f-19dc0b7f0157 kind: Event lastTimestamp: "2026-07-20T10:09:02Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1771" metadata: creationTimestamp: "2026-07-20T10:09:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:09:02Z" name: error-404-isvc-ab339-predictor-98847c874-24mfc.18c3f79fb5fa5cfc namespace: kserve-ci-e2e-test resourceVersion: "15025" uid: 468993dd-1bb7-4639-a71e-140bcd312d4b reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ab339-predictor-98847c874-24mfc namespace: kserve-ci-e2e-test resourceVersion: "14944" uid: 6633df2b-88a5-450c-875f-19dc0b7f0157 kind: Event lastTimestamp: "2026-07-20T10:09:19Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1771" in 16.722s (16.722s including waiting). Image size: 1334535787 bytes.' metadata: creationTimestamp: "2026-07-20T10:09:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:09:19Z" name: error-404-isvc-ab339-predictor-98847c874-24mfc.18c3f7a39ab373d8 namespace: kserve-ci-e2e-test resourceVersion: "15180" uid: a4420b5c-1620-4b2c-9435-4b9359e2c3f3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ab339-predictor-98847c874-24mfc namespace: kserve-ci-e2e-test resourceVersion: "14944" uid: 6633df2b-88a5-450c-875f-19dc0b7f0157 kind: Event lastTimestamp: "2026-07-20T10:09:19Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-20T10:09:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:09:19Z" name: error-404-isvc-ab339-predictor-98847c874-24mfc.18c3f7a39ef6e579 namespace: kserve-ci-e2e-test resourceVersion: "15183" uid: 506e766e-7c62-416a-ad1f-89d5cec9b5c4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ab339-predictor-98847c874-24mfc namespace: kserve-ci-e2e-test resourceVersion: "14944" uid: 6633df2b-88a5-450c-875f-19dc0b7f0157 kind: Event lastTimestamp: "2026-07-20T10:09:19Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-20T10:09:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:09:19Z" name: error-404-isvc-ab339-predictor-98847c874-24mfc.18c3f7a39f6dec50 namespace: kserve-ci-e2e-test resourceVersion: "15185" uid: d0d6abe8-ce68-43dd-b5b0-74686debb132 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ab339-predictor-98847c874-24mfc namespace: kserve-ci-e2e-test resourceVersion: "14944" uid: 6633df2b-88a5-450c-875f-19dc0b7f0157 kind: Event lastTimestamp: "2026-07-20T10:09:19Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-20T10:09:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:09:19Z" name: error-404-isvc-ab339-predictor-98847c874-24mfc.18c3f7a39f7c0f89 namespace: kserve-ci-e2e-test resourceVersion: "15186" uid: bf343587-4373-4838-a933-470be92f3ae0 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ab339-predictor-98847c874-24mfc namespace: kserve-ci-e2e-test resourceVersion: "14944" uid: 6633df2b-88a5-450c-875f-19dc0b7f0157 kind: Event lastTimestamp: "2026-07-20T10:09:22Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.51s (2.51s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-20T10:09:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:09:22Z" name: error-404-isvc-ab339-predictor-98847c874-24mfc.18c3f7a4351dcc1c namespace: kserve-ci-e2e-test resourceVersion: "15210" uid: e951ca94-e1e7-4d6f-bf04-002bd309a14c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ab339-predictor-98847c874-24mfc namespace: kserve-ci-e2e-test resourceVersion: "14944" uid: 6633df2b-88a5-450c-875f-19dc0b7f0157 kind: Event lastTimestamp: "2026-07-20T10:09:22Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-20T10:09:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:09:22Z" name: error-404-isvc-ab339-predictor-98847c874-24mfc.18c3f7a438e0eec5 namespace: kserve-ci-e2e-test resourceVersion: "15213" uid: dc52d262-957f-4af9-80b2-ea396b055e55 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ab339-predictor-98847c874-24mfc namespace: kserve-ci-e2e-test resourceVersion: "14944" uid: 6633df2b-88a5-450c-875f-19dc0b7f0157 kind: Event lastTimestamp: "2026-07-20T10:09:22Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:09:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:09:22Z" name: error-404-isvc-ab339-predictor-98847c874-24mfc.18c3f7a4394d3ea4 namespace: kserve-ci-e2e-test resourceVersion: "15215" uid: da6c536b-b08f-4aa7-a807-dab58039f5ea reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-20T10:09:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ab339-predictor-98847c874-24mfc namespace: kserve-ci-e2e-test resourceVersion: "14944" uid: 6633df2b-88a5-450c-875f-19dc0b7f0157 kind: Event lastTimestamp: "2026-07-20T10:09:58Z" message: 'Readiness probe failed: dial tcp 10.133.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-20T10:09:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:09:58Z" name: error-404-isvc-ab339-predictor-98847c874-24mfc.18c3f7a44d03b6d4 namespace: kserve-ci-e2e-test resourceVersion: "15546" uid: 93e5ddc1-e8a2-48a0-8b31-f6862e6ba977 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ab339-predictor-98847c874-24mfc namespace: kserve-ci-e2e-test resourceVersion: "14944" uid: 6633df2b-88a5-450c-875f-19dc0b7f0157 kind: Event lastTimestamp: "2026-07-20T10:10:31Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-20T10:10:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:10:31Z" name: error-404-isvc-ab339-predictor-98847c874-24mfc.18c3f7b45ad5bf9f namespace: kserve-ci-e2e-test resourceVersion: "15929" uid: b02bf8db-c83f-4d8e-af82-1a2e18191d3b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ab339-predictor-98847c874-24mfc namespace: kserve-ci-e2e-test resourceVersion: "14944" uid: 6633df2b-88a5-450c-875f-19dc0b7f0157 kind: Event lastTimestamp: "2026-07-20T10:10:31Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:10:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:10:31Z" name: error-404-isvc-ab339-predictor-98847c874-24mfc.18c3f7b45ad94114 namespace: kserve-ci-e2e-test resourceVersion: "15933" uid: ad0d46fd-4787-439e-adba-5cb0bd75c808 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ab339-predictor-98847c874-24mfc namespace: kserve-ci-e2e-test resourceVersion: "14944" uid: 6633df2b-88a5-450c-875f-19dc0b7f0157 kind: Event lastTimestamp: "2026-07-20T10:10:33Z" message: 'Readiness probe failed: Get "https://10.133.0.37:8643/healthz": dial tcp 10.133.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-20T10:10:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:10:33Z" name: error-404-isvc-ab339-predictor-98847c874-24mfc.18c3f7b4d5608961 namespace: kserve-ci-e2e-test resourceVersion: "16030" uid: 60e6e896-843c-453e-95be-3f9605215aae reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-ab339-predictor-98847c874 namespace: kserve-ci-e2e-test resourceVersion: "14936" uid: 652b5dcc-d709-4372-b9b4-06921755e44e kind: Event lastTimestamp: "2026-07-20T10:09:01Z" message: 'Created pod: error-404-isvc-ab339-predictor-98847c874-24mfc' metadata: creationTimestamp: "2026-07-20T10:09:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:09:01Z" name: error-404-isvc-ab339-predictor-98847c874.18c3f79f747e3627 namespace: kserve-ci-e2e-test resourceVersion: "14943" uid: 1c24e727-0093-46ef-8764-0c18dc95f680 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-ab339-predictor namespace: kserve-ci-e2e-test resourceVersion: "14933" uid: c772af36-77d7-4b70-a201-be9a3c87944d kind: Event lastTimestamp: "2026-07-20T10:09:01Z" message: Scaled up replica set error-404-isvc-ab339-predictor-98847c874 from 0 to 1 metadata: creationTimestamp: "2026-07-20T10:09:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:09:01Z" name: error-404-isvc-ab339-predictor.18c3f79f72c34f43 namespace: kserve-ci-e2e-test resourceVersion: "14938" uid: 7073e61e-bea2-47a1-a2df-bdd1a4e15ae4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-20T10:09:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ab339-predictor namespace: kserve-ci-e2e-test resourceVersion: "14935" uid: add55a60-ed73-4810-a889-64ae938075f9 kind: Event lastTimestamp: "2026-07-20T10:09:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:09:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:09:46Z" name: error-404-isvc-ab339-predictor.18c3f7a2f139de6e namespace: kserve-ci-e2e-test resourceVersion: "15456" uid: e2a77aee-bde4-4e84-9f3b-04f98cc1a35a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-20T10:09:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ab339-predictor namespace: kserve-ci-e2e-test resourceVersion: "14935" uid: add55a60-ed73-4810-a889-64ae938075f9 kind: Event lastTimestamp: "2026-07-20T10:09:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:09:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:09:46Z" name: error-404-isvc-ab339-predictor.18c3f7a2f1724ce6 namespace: kserve-ci-e2e-test resourceVersion: "15457" uid: 16f58ff6-1bc7-42ea-ac6a-56846b75ff63 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-20T10:10:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ab339-predictor namespace: kserve-ci-e2e-test resourceVersion: "15153" uid: add55a60-ed73-4810-a889-64ae938075f9 kind: Event lastTimestamp: "2026-07-20T10:10:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:10:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:10:16Z" name: error-404-isvc-ab339-predictor.18c3f7ad6dcf8cf5 namespace: kserve-ci-e2e-test resourceVersion: "15709" uid: 6dda637b-3bfb-45d2-990a-a4b8661cd7ad reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-20T10:10:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ab339-predictor namespace: kserve-ci-e2e-test resourceVersion: "15153" uid: add55a60-ed73-4810-a889-64ae938075f9 kind: Event lastTimestamp: "2026-07-20T10:10:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:10:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:10:16Z" name: error-404-isvc-ab339-predictor.18c3f7ad6e7cafbd namespace: kserve-ci-e2e-test resourceVersion: "15710" uid: 5858bc61-81b9-44ad-bb0f-ce0a06926015 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-20T10:09:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ab339 namespace: kserve-ci-e2e-test resourceVersion: "14916" uid: 54c94ba3-9051-4780-85da-d67bcbf30b95 kind: Event lastTimestamp: "2026-07-20T10:10:08Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-20T10:09:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:10:08Z" name: error-404-isvc-ab339.18c3f79f72bed125 namespace: kserve-ci-e2e-test resourceVersion: "15646" uid: 6b3f3e3b-eb12-4211-9eff-8879b07feb9a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ab339 namespace: kserve-ci-e2e-test resourceVersion: "15641" uid: 54c94ba3-9051-4780-85da-d67bcbf30b95 kind: Event lastTimestamp: "2026-07-20T10:10:08Z" message: InferenceService [error-404-isvc-ab339] is Ready metadata: creationTimestamp: "2026-07-20T10:10:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:10:08Z" name: error-404-isvc-ab339.18c3f7af08ec8fdb namespace: kserve-ci-e2e-test resourceVersion: "15642" uid: 29cb2b7a-487e-42c4-94bf-558e5737f69f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-20T10:09:01.199756Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-7669b84dcc-69zjp namespace: kserve-ci-e2e-test resourceVersion: "14862" uid: 742f0c97-d4c0-4472-b4c6-5693c3f4c673 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-7669b84dcc-69zjp to ip-10-0-143-155.ec2.internal metadata: creationTimestamp: "2026-07-20T10:09:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-20T10:09:01Z" name: isvc-sklearn-graph-1-predictor-7669b84dcc-69zjp.18c3f79f5ced4659 namespace: kserve-ci-e2e-test resourceVersion: "14867" uid: cdadff7a-463d-4004-8fc8-b832891381c5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5dc669d9-69hkz source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:01Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-7669b84dcc-69zjp namespace: kserve-ci-e2e-test resourceVersion: "14871" uid: 742f0c97-d4c0-4472-b4c6-5693c3f4c673 kind: Event lastTimestamp: "2026-07-20T10:09:01Z" message: Add eth0 [10.133.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-20T10:09:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-20T10:09:01Z" name: isvc-sklearn-graph-1-predictor-7669b84dcc-69zjp.18c3f79f774eadc3 namespace: kserve-ci-e2e-test resourceVersion: "14956" uid: 6b86fa5c-88bb-45c8-8b70-4ed3b7258dcb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-7669b84dcc-69zjp namespace: kserve-ci-e2e-test resourceVersion: "14865" uid: 742f0c97-d4c0-4472-b4c6-5693c3f4c673 kind: Event lastTimestamp: "2026-07-20T10:09:01Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:038cd29f3e031cdfd5fa54ba896bbecc69cd255eda6b470c2f7bce5282357bb8" metadata: creationTimestamp: "2026-07-20T10:09:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:09:01Z" name: isvc-sklearn-graph-1-predictor-7669b84dcc-69zjp.18c3f79f79234177 namespace: kserve-ci-e2e-test resourceVersion: "14965" uid: 5b90a99e-e66a-46f1-9e61-fdc1397adaaa reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:17Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-7669b84dcc-69zjp namespace: kserve-ci-e2e-test resourceVersion: "14865" uid: 742f0c97-d4c0-4472-b4c6-5693c3f4c673 kind: Event lastTimestamp: "2026-07-20T10:09:17Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:038cd29f3e031cdfd5fa54ba896bbecc69cd255eda6b470c2f7bce5282357bb8" in 16.065s (16.065s including waiting). Image size: 300593107 bytes.' metadata: creationTimestamp: "2026-07-20T10:09:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:09:17Z" name: isvc-sklearn-graph-1-predictor-7669b84dcc-69zjp.18c3f7a336b8b68a namespace: kserve-ci-e2e-test resourceVersion: "15166" uid: a896e127-236b-4cb1-98ec-ad0ccf207418 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:19Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-7669b84dcc-69zjp namespace: kserve-ci-e2e-test resourceVersion: "14865" uid: 742f0c97-d4c0-4472-b4c6-5693c3f4c673 kind: Event lastTimestamp: "2026-07-20T10:09:19Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-20T10:09:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:09:19Z" name: isvc-sklearn-graph-1-predictor-7669b84dcc-69zjp.18c3f7a39b339a75 namespace: kserve-ci-e2e-test resourceVersion: "15181" uid: 5b44349d-dfe0-4f11-ac3f-f546eee9ea79 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:19Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-7669b84dcc-69zjp namespace: kserve-ci-e2e-test resourceVersion: "14865" uid: 742f0c97-d4c0-4472-b4c6-5693c3f4c673 kind: Event lastTimestamp: "2026-07-20T10:09:19Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-20T10:09:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:09:19Z" name: isvc-sklearn-graph-1-predictor-7669b84dcc-69zjp.18c3f7a39bb54a1c namespace: kserve-ci-e2e-test resourceVersion: "15182" uid: e8b78d0e-70c9-4615-88fd-40c5b0211cea reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7669b84dcc-69zjp namespace: kserve-ci-e2e-test resourceVersion: "14865" uid: 742f0c97-d4c0-4472-b4c6-5693c3f4c673 kind: Event lastTimestamp: "2026-07-20T10:09:23Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1771" metadata: creationTimestamp: "2026-07-20T10:09:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:09:23Z" name: isvc-sklearn-graph-1-predictor-7669b84dcc-69zjp.18c3f7a48914b34b namespace: kserve-ci-e2e-test resourceVersion: "15236" uid: e34fb9e6-5d6e-4c68-a692-c7d151763d0a reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7669b84dcc-69zjp namespace: kserve-ci-e2e-test resourceVersion: "14865" uid: 742f0c97-d4c0-4472-b4c6-5693c3f4c673 kind: Event lastTimestamp: "2026-07-20T10:09:29Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1771" in 6.153s (6.153s including waiting). Image size: 1555845555 bytes.' metadata: creationTimestamp: "2026-07-20T10:09:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:09:29Z" name: isvc-sklearn-graph-1-predictor-7669b84dcc-69zjp.18c3f7a5f7df3247 namespace: kserve-ci-e2e-test resourceVersion: "15288" uid: 5d750f86-e157-4144-936e-c9d5a61719af reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7669b84dcc-69zjp namespace: kserve-ci-e2e-test resourceVersion: "14865" uid: 742f0c97-d4c0-4472-b4c6-5693c3f4c673 kind: Event lastTimestamp: "2026-07-20T10:09:29Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-20T10:09:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:09:29Z" name: isvc-sklearn-graph-1-predictor-7669b84dcc-69zjp.18c3f7a5fb8f727d namespace: kserve-ci-e2e-test resourceVersion: "15290" uid: 32c01e8a-45ee-420c-85a3-825f0ec747ab reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7669b84dcc-69zjp namespace: kserve-ci-e2e-test resourceVersion: "14865" uid: 742f0c97-d4c0-4472-b4c6-5693c3f4c673 kind: Event lastTimestamp: "2026-07-20T10:09:29Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-20T10:09:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:09:29Z" name: isvc-sklearn-graph-1-predictor-7669b84dcc-69zjp.18c3f7a5fc005965 namespace: kserve-ci-e2e-test resourceVersion: "15291" uid: 8f72d735-af42-430c-80c0-c12733f23c15 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-7669b84dcc-69zjp namespace: kserve-ci-e2e-test resourceVersion: "14865" uid: 742f0c97-d4c0-4472-b4c6-5693c3f4c673 kind: Event lastTimestamp: "2026-07-20T10:09:29Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-20T10:09:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:09:29Z" name: isvc-sklearn-graph-1-predictor-7669b84dcc-69zjp.18c3f7a5fc13adae namespace: kserve-ci-e2e-test resourceVersion: "15292" uid: f51d0b3f-acec-49c6-a493-aaccab2eb5a1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-7669b84dcc-69zjp namespace: kserve-ci-e2e-test resourceVersion: "14865" uid: 742f0c97-d4c0-4472-b4c6-5693c3f4c673 kind: Event lastTimestamp: "2026-07-20T10:09:29Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-20T10:09:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:09:29Z" name: isvc-sklearn-graph-1-predictor-7669b84dcc-69zjp.18c3f7a5ff200652 namespace: kserve-ci-e2e-test resourceVersion: "15294" uid: 3087e89a-d61a-43eb-8475-b398eb47717d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-7669b84dcc-69zjp namespace: kserve-ci-e2e-test resourceVersion: "14865" uid: 742f0c97-d4c0-4472-b4c6-5693c3f4c673 kind: Event lastTimestamp: "2026-07-20T10:09:29Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:09:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:09:29Z" name: isvc-sklearn-graph-1-predictor-7669b84dcc-69zjp.18c3f7a5ff903141 namespace: kserve-ci-e2e-test resourceVersion: "15295" uid: 9dc47649-43ea-475e-b1aa-9521de6f3a0c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-20T10:09:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7669b84dcc-69zjp namespace: kserve-ci-e2e-test resourceVersion: "14865" uid: 742f0c97-d4c0-4472-b4c6-5693c3f4c673 kind: Event lastTimestamp: "2026-07-20T10:10:36Z" message: 'Readiness probe failed: dial tcp 10.133.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-20T10:09:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:10:36Z" name: isvc-sklearn-graph-1-predictor-7669b84dcc-69zjp.18c3f7a62ca36a88 namespace: kserve-ci-e2e-test resourceVersion: "16061" uid: c7479d30-83d8-493e-88e8-df48f4b2b554 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7669b84dcc-69zjp namespace: kserve-ci-e2e-test resourceVersion: "14865" uid: 742f0c97-d4c0-4472-b4c6-5693c3f4c673 kind: Event lastTimestamp: "2026-07-20T10:11:21Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-20T10:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:11:21Z" name: isvc-sklearn-graph-1-predictor-7669b84dcc-69zjp.18c3f7c00141364f namespace: kserve-ci-e2e-test resourceVersion: "16548" uid: 58436092-35dd-4b43-a06b-3b70069c8bdb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-7669b84dcc-69zjp namespace: kserve-ci-e2e-test resourceVersion: "14865" uid: 742f0c97-d4c0-4472-b4c6-5693c3f4c673 kind: Event lastTimestamp: "2026-07-20T10:11:21Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:11:21Z" name: isvc-sklearn-graph-1-predictor-7669b84dcc-69zjp.18c3f7c00141be07 namespace: kserve-ci-e2e-test resourceVersion: "16552" uid: 103a1199-6f2f-415b-a281-967cbf9e8845 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-7669b84dcc-69zjp namespace: kserve-ci-e2e-test resourceVersion: "14865" uid: 742f0c97-d4c0-4472-b4c6-5693c3f4c673 kind: Event lastTimestamp: "2026-07-20T10:11:21Z" message: 'Readiness probe failed: Get "https://10.133.0.35:8643/healthz": dial tcp 10.133.0.35:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-20T10:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:11:21Z" name: isvc-sklearn-graph-1-predictor-7669b84dcc-69zjp.18c3f7c0050ac607 namespace: kserve-ci-e2e-test resourceVersion: "16577" uid: 30527c0f-e6c5-4fc7-9bda-507ca105aea9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-7669b84dcc namespace: kserve-ci-e2e-test resourceVersion: "14850" uid: d05c0abf-6986-442b-b836-234c142e02ea kind: Event lastTimestamp: "2026-07-20T10:09:01Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-7669b84dcc-69zjp' metadata: creationTimestamp: "2026-07-20T10:09:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:09:01Z" name: isvc-sklearn-graph-1-predictor-7669b84dcc.18c3f79f5c39bb2d namespace: kserve-ci-e2e-test resourceVersion: "14863" uid: 87bba2c7-e778-4180-8c3c-99a4f5d60595 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14849" uid: f3e2da09-be8d-499a-9cb1-465ace27098f kind: Event lastTimestamp: "2026-07-20T10:09:01Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-7669b84dcc from 0 to 1 metadata: creationTimestamp: "2026-07-20T10:09:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:09:01Z" name: isvc-sklearn-graph-1-predictor.18c3f79f547c7e48 namespace: kserve-ci-e2e-test resourceVersion: "14851" uid: 7a17567c-f535-4ca2-a48c-628615c06bda reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-20T10:09:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14857" uid: 0be31899-5420-4472-97be-7cc1d378e9b3 kind: Event lastTimestamp: "2026-07-20T10:10:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:09:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:10:01Z" name: isvc-sklearn-graph-1-predictor.18c3f7a2d9b911f9 namespace: kserve-ci-e2e-test resourceVersion: "15571" uid: 6ac95f17-fce0-4e2c-869b-aae7831d37ec reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-20T10:09:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14857" uid: 0be31899-5420-4472-97be-7cc1d378e9b3 kind: Event lastTimestamp: "2026-07-20T10:10:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:09:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:10:01Z" name: isvc-sklearn-graph-1-predictor.18c3f7a2da241f64 namespace: kserve-ci-e2e-test resourceVersion: "15572" uid: f014827d-a265-46eb-ab6a-1632271b0fd1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-20T10:10:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15142" uid: 0be31899-5420-4472-97be-7cc1d378e9b3 kind: Event lastTimestamp: "2026-07-20T10:11:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:10:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:11:16Z" name: isvc-sklearn-graph-1-predictor.18c3f7b0d6c5c411 namespace: kserve-ci-e2e-test resourceVersion: "16433" uid: 0e35cec7-307e-49ae-80b4-02bb20f10015 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-20T10:10:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15142" uid: 0be31899-5420-4472-97be-7cc1d378e9b3 kind: Event lastTimestamp: "2026-07-20T10:11:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:10:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:11:16Z" name: isvc-sklearn-graph-1-predictor.18c3f7b0d72913a7 namespace: kserve-ci-e2e-test resourceVersion: "16434" uid: 2bf23ae0-9c91-43d8-b81a-f274b72f53e6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-20T10:09:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14837" uid: 942a5761-062f-48a6-9e8c-10f5adfaf3e3 kind: Event lastTimestamp: "2026-07-20T10:10:46Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-20T10:09:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:10:46Z" name: isvc-sklearn-graph-1.18c3f79f5a47cb39 namespace: kserve-ci-e2e-test resourceVersion: "16158" uid: ecd53074-24eb-4375-bbf5-401ecfa38b73 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "16152" uid: 942a5761-062f-48a6-9e8c-10f5adfaf3e3 kind: Event lastTimestamp: "2026-07-20T10:10:46Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-07-20T10:10:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:10:46Z" name: isvc-sklearn-graph-1.18c3f7b7e699963b namespace: kserve-ci-e2e-test resourceVersion: "16156" uid: 26373f3b-859b-408c-964a-444227ced859 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-20T10:09:01.803429Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-56c9f5f74c-74ttp namespace: kserve-ci-e2e-test resourceVersion: "14976" uid: 878bcff2-8ba5-4aa6-b489-6e28977a959d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-56c9f5f74c-74ttp to ip-10-0-133-245.ec2.internal metadata: creationTimestamp: "2026-07-20T10:09:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-20T10:09:01Z" name: isvc-sklearn-graph-2-predictor-56c9f5f74c-74ttp.18c3f79f80e89235 namespace: kserve-ci-e2e-test resourceVersion: "14980" uid: 6598569f-17a7-4646-ba97-bc5582f8ad9a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5dc669d9-69hkz source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:02Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-56c9f5f74c-74ttp namespace: kserve-ci-e2e-test resourceVersion: "14984" uid: 878bcff2-8ba5-4aa6-b489-6e28977a959d kind: Event lastTimestamp: "2026-07-20T10:09:02Z" message: Add eth0 [10.132.0.18/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-20T10:09:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-20T10:09:02Z" name: isvc-sklearn-graph-2-predictor-56c9f5f74c-74ttp.18c3f79f9ab82aa7 namespace: kserve-ci-e2e-test resourceVersion: "15010" uid: 87f936a1-5e00-450d-9914-f4332aabeea9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-56c9f5f74c-74ttp namespace: kserve-ci-e2e-test resourceVersion: "14978" uid: 878bcff2-8ba5-4aa6-b489-6e28977a959d kind: Event lastTimestamp: "2026-07-20T10:09:02Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:038cd29f3e031cdfd5fa54ba896bbecc69cd255eda6b470c2f7bce5282357bb8" already present on machine metadata: creationTimestamp: "2026-07-20T10:09:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:09:02Z" name: isvc-sklearn-graph-2-predictor-56c9f5f74c-74ttp.18c3f79f9c335bdf namespace: kserve-ci-e2e-test resourceVersion: "15013" uid: d45cbc55-c97e-4969-bfd9-3ab09cfd9bb2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-245.ec2.internal source: component: kubelet host: ip-10-0-133-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-56c9f5f74c-74ttp namespace: kserve-ci-e2e-test resourceVersion: "14978" uid: 878bcff2-8ba5-4aa6-b489-6e28977a959d kind: Event lastTimestamp: "2026-07-20T10:09:02Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-20T10:09:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:09:02Z" name: isvc-sklearn-graph-2-predictor-56c9f5f74c-74ttp.18c3f79f9f3d4339 namespace: kserve-ci-e2e-test resourceVersion: "15014" uid: f7b8ebfc-c93d-4f3e-9eba-29264ebf2028 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-245.ec2.internal source: component: kubelet host: ip-10-0-133-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-56c9f5f74c-74ttp namespace: kserve-ci-e2e-test resourceVersion: "14978" uid: 878bcff2-8ba5-4aa6-b489-6e28977a959d kind: Event lastTimestamp: "2026-07-20T10:09:02Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-20T10:09:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:09:02Z" name: isvc-sklearn-graph-2-predictor-56c9f5f74c-74ttp.18c3f79f9fbd6092 namespace: kserve-ci-e2e-test resourceVersion: "15015" uid: f805578b-b12e-4e78-9088-a30be99eac72 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-245.ec2.internal source: component: kubelet host: ip-10-0-133-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-56c9f5f74c-74ttp namespace: kserve-ci-e2e-test resourceVersion: "14978" uid: 878bcff2-8ba5-4aa6-b489-6e28977a959d kind: Event lastTimestamp: "2026-07-20T10:09:05Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1771" metadata: creationTimestamp: "2026-07-20T10:09:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:09:05Z" name: isvc-sklearn-graph-2-predictor-56c9f5f74c-74ttp.18c3f7a0794e45d2 namespace: kserve-ci-e2e-test resourceVersion: "15064" uid: 79d9c430-74df-4752-92b3-363f74b7040b reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-245.ec2.internal source: component: kubelet host: ip-10-0-133-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-56c9f5f74c-74ttp namespace: kserve-ci-e2e-test resourceVersion: "14978" uid: 878bcff2-8ba5-4aa6-b489-6e28977a959d kind: Event lastTimestamp: "2026-07-20T10:09:29Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1771" in 23.865s (23.865s including waiting). Image size: 1555845555 bytes.' metadata: creationTimestamp: "2026-07-20T10:09:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:09:29Z" name: isvc-sklearn-graph-2-predictor-56c9f5f74c-74ttp.18c3f7a607c8d9cf namespace: kserve-ci-e2e-test resourceVersion: "15297" uid: 70563388-bdd4-4ead-ab0e-c14442a4b891 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-245.ec2.internal source: component: kubelet host: ip-10-0-133-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-56c9f5f74c-74ttp namespace: kserve-ci-e2e-test resourceVersion: "14978" uid: 878bcff2-8ba5-4aa6-b489-6e28977a959d kind: Event lastTimestamp: "2026-07-20T10:09:29Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-20T10:09:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:09:29Z" name: isvc-sklearn-graph-2-predictor-56c9f5f74c-74ttp.18c3f7a60e01a7b7 namespace: kserve-ci-e2e-test resourceVersion: "15298" uid: f6bbba44-3938-4c79-8ade-f317e10ccd57 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-245.ec2.internal source: component: kubelet host: ip-10-0-133-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-56c9f5f74c-74ttp namespace: kserve-ci-e2e-test resourceVersion: "14978" uid: 878bcff2-8ba5-4aa6-b489-6e28977a959d kind: Event lastTimestamp: "2026-07-20T10:09:29Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-20T10:09:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:09:29Z" name: isvc-sklearn-graph-2-predictor-56c9f5f74c-74ttp.18c3f7a60ec8db71 namespace: kserve-ci-e2e-test resourceVersion: "15299" uid: 7ef5105e-1d93-4a96-a573-0759b56804ab reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-245.ec2.internal source: component: kubelet host: ip-10-0-133-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-56c9f5f74c-74ttp namespace: kserve-ci-e2e-test resourceVersion: "14978" uid: 878bcff2-8ba5-4aa6-b489-6e28977a959d kind: Event lastTimestamp: "2026-07-20T10:09:29Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-20T10:09:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:09:29Z" name: isvc-sklearn-graph-2-predictor-56c9f5f74c-74ttp.18c3f7a60ed732bc namespace: kserve-ci-e2e-test resourceVersion: "15300" uid: 497d5787-8800-4166-a54c-2eb678e8bc9f reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-245.ec2.internal source: component: kubelet host: ip-10-0-133-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-56c9f5f74c-74ttp namespace: kserve-ci-e2e-test resourceVersion: "14978" uid: 878bcff2-8ba5-4aa6-b489-6e28977a959d kind: Event lastTimestamp: "2026-07-20T10:09:32Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 3.033s (3.033s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-20T10:09:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:09:32Z" name: isvc-sklearn-graph-2-predictor-56c9f5f74c-74ttp.18c3f7a6c3a4fd2f namespace: kserve-ci-e2e-test resourceVersion: "15341" uid: 1f7f170f-c53d-4ac3-ad9f-bf2f7449b461 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-245.ec2.internal source: component: kubelet host: ip-10-0-133-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-56c9f5f74c-74ttp namespace: kserve-ci-e2e-test resourceVersion: "14978" uid: 878bcff2-8ba5-4aa6-b489-6e28977a959d kind: Event lastTimestamp: "2026-07-20T10:09:33Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-20T10:09:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:09:33Z" name: isvc-sklearn-graph-2-predictor-56c9f5f74c-74ttp.18c3f7a6ce4c1c9d namespace: kserve-ci-e2e-test resourceVersion: "15347" uid: 48e49746-327b-4a87-9c5a-22cb0774af3d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-245.ec2.internal source: component: kubelet host: ip-10-0-133-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-56c9f5f74c-74ttp namespace: kserve-ci-e2e-test resourceVersion: "14978" uid: 878bcff2-8ba5-4aa6-b489-6e28977a959d kind: Event lastTimestamp: "2026-07-20T10:09:33Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:09:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:09:33Z" name: isvc-sklearn-graph-2-predictor-56c9f5f74c-74ttp.18c3f7a6cf81a6af namespace: kserve-ci-e2e-test resourceVersion: "15348" uid: 3249fad1-9424-47c1-a407-b5deedc54e0f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-245.ec2.internal source: component: kubelet host: ip-10-0-133-245.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-20T10:09:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-56c9f5f74c-74ttp namespace: kserve-ci-e2e-test resourceVersion: "14978" uid: 878bcff2-8ba5-4aa6-b489-6e28977a959d kind: Event lastTimestamp: "2026-07-20T10:10:40Z" message: 'Readiness probe failed: dial tcp 10.132.0.18:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-20T10:09:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:10:40Z" name: isvc-sklearn-graph-2-predictor-56c9f5f74c-74ttp.18c3f7a706fc2bb1 namespace: kserve-ci-e2e-test resourceVersion: "16090" uid: c19914ec-65c7-43bc-a4ae-73b354d11e41 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-245.ec2.internal source: component: kubelet host: ip-10-0-133-245.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-56c9f5f74c-74ttp namespace: kserve-ci-e2e-test resourceVersion: "14978" uid: 878bcff2-8ba5-4aa6-b489-6e28977a959d kind: Event lastTimestamp: "2026-07-20T10:11:21Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-20T10:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:11:21Z" name: isvc-sklearn-graph-2-predictor-56c9f5f74c-74ttp.18c3f7c00b6f3548 namespace: kserve-ci-e2e-test resourceVersion: "16619" uid: 6c13bc72-adf1-4bf5-99e7-c88d9af56e1a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-245.ec2.internal source: component: kubelet host: ip-10-0-133-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-56c9f5f74c-74ttp namespace: kserve-ci-e2e-test resourceVersion: "14978" uid: 878bcff2-8ba5-4aa6-b489-6e28977a959d kind: Event lastTimestamp: "2026-07-20T10:11:21Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:11:21Z" name: isvc-sklearn-graph-2-predictor-56c9f5f74c-74ttp.18c3f7c00b6fc866 namespace: kserve-ci-e2e-test resourceVersion: "16620" uid: 1b8bc874-0e19-4bf2-bb96-c904c2f385fe reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-245.ec2.internal source: component: kubelet host: ip-10-0-133-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-56c9f5f74c-74ttp namespace: kserve-ci-e2e-test resourceVersion: "14978" uid: 878bcff2-8ba5-4aa6-b489-6e28977a959d kind: Event lastTimestamp: "2026-07-20T10:11:25Z" message: 'Readiness probe failed: Get "https://10.132.0.18:8643/healthz": dial tcp 10.132.0.18:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-20T10:11:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:11:25Z" name: isvc-sklearn-graph-2-predictor-56c9f5f74c-74ttp.18c3f7c0df7ac590 namespace: kserve-ci-e2e-test resourceVersion: "16681" uid: 25e0e9af-47ab-4b94-9eef-7c0de3adb9dd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-245.ec2.internal source: component: kubelet host: ip-10-0-133-245.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-56c9f5f74c namespace: kserve-ci-e2e-test resourceVersion: "14964" uid: d183865b-467a-4850-ae69-0c02fc79260c kind: Event lastTimestamp: "2026-07-20T10:09:01Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-56c9f5f74c-74ttp' metadata: creationTimestamp: "2026-07-20T10:09:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:09:01Z" name: isvc-sklearn-graph-2-predictor-56c9f5f74c.18c3f79f8007dfb5 namespace: kserve-ci-e2e-test resourceVersion: "14977" uid: 780049a6-85e2-4f10-bf1b-319b9887f484 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14960" uid: e33e0564-93be-4bfc-a630-5ee5b68670c6 kind: Event lastTimestamp: "2026-07-20T10:09:01Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-56c9f5f74c from 0 to 1 metadata: creationTimestamp: "2026-07-20T10:09:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:09:01Z" name: isvc-sklearn-graph-2-predictor.18c3f79f794c5659 namespace: kserve-ci-e2e-test resourceVersion: "14968" uid: 4b768c51-3125-4f2f-80bb-53bc274f8516 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-20T10:09:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14962" uid: 1d90c01f-a766-46d7-9de6-0251ebe8220b kind: Event lastTimestamp: "2026-07-20T10:10:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:09:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:10:01Z" name: isvc-sklearn-graph-2-predictor.18c3f7a2f7976ffe namespace: kserve-ci-e2e-test resourceVersion: "15584" uid: 9c5b4e67-1422-4595-b59c-3aca77f2ba30 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-20T10:09:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14962" uid: 1d90c01f-a766-46d7-9de6-0251ebe8220b kind: Event lastTimestamp: "2026-07-20T10:10:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:09:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:10:01Z" name: isvc-sklearn-graph-2-predictor.18c3f7a2f7fe9939 namespace: kserve-ci-e2e-test resourceVersion: "15585" uid: 99a2a08f-fbb2-43d2-977d-c83ef6fc902a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-20T10:10:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15157" uid: 1d90c01f-a766-46d7-9de6-0251ebe8220b kind: Event lastTimestamp: "2026-07-20T10:11:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:10:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:11:16Z" name: isvc-sklearn-graph-2-predictor.18c3f7b0f2f436c7 namespace: kserve-ci-e2e-test resourceVersion: "16445" uid: 872dd4eb-4f4d-4f66-aad3-b164ddc84297 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-20T10:10:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15157" uid: 1d90c01f-a766-46d7-9de6-0251ebe8220b kind: Event lastTimestamp: "2026-07-20T10:11:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:10:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:11:16Z" name: isvc-sklearn-graph-2-predictor.18c3f7b0f35ec4bb namespace: kserve-ci-e2e-test resourceVersion: "16446" uid: 0c720e54-7ebc-4aeb-b5c0-c732b745f043 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-20T10:09:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14946" uid: 8a15bd9b-41b5-41b4-802a-c0ebd0082741 kind: Event lastTimestamp: "2026-07-20T10:10:50Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-20T10:09:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:10:50Z" name: isvc-sklearn-graph-2.18c3f79f794ab379 namespace: kserve-ci-e2e-test resourceVersion: "16198" uid: 2dbb5385-4859-4925-88d7-f22c8b086ba6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "16195" uid: 8a15bd9b-41b5-41b4-802a-c0ebd0082741 kind: Event lastTimestamp: "2026-07-20T10:10:50Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-07-20T10:10:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:10:50Z" name: isvc-sklearn-graph-2.18c3f7b8bf077045 namespace: kserve-ci-e2e-test resourceVersion: "16196" uid: d8461ff8-7e4b-4c02-af1a-f35121e8ce21 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-20T10:09:01.508493Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ldfq2 namespace: kserve-ci-e2e-test resourceVersion: "14917" uid: 195ca154-8722-4be5-bf3f-faafda88d097 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-555fc69d56-ldfq2 to ip-10-0-133-245.ec2.internal metadata: creationTimestamp: "2026-07-20T10:09:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-20T10:09:01Z" name: isvc-xgboost-graph-predictor-555fc69d56-ldfq2.18c3f79f6f5437ee namespace: kserve-ci-e2e-test resourceVersion: "14922" uid: 6a77a7dd-fa64-4c6e-9eee-1e88d5bb3a6f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5dc669d9-69hkz source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:01Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ldfq2 namespace: kserve-ci-e2e-test resourceVersion: "14930" uid: 195ca154-8722-4be5-bf3f-faafda88d097 kind: Event lastTimestamp: "2026-07-20T10:09:01Z" message: Add eth0 [10.132.0.17/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-20T10:09:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-20T10:09:01Z" name: isvc-xgboost-graph-predictor-555fc69d56-ldfq2.18c3f79f88c2324a namespace: kserve-ci-e2e-test resourceVersion: "14996" uid: 2afab199-6f39-446b-9bbf-47fe52f55f47 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ldfq2 namespace: kserve-ci-e2e-test resourceVersion: "14919" uid: 195ca154-8722-4be5-bf3f-faafda88d097 kind: Event lastTimestamp: "2026-07-20T10:09:01Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:038cd29f3e031cdfd5fa54ba896bbecc69cd255eda6b470c2f7bce5282357bb8" already present on machine metadata: creationTimestamp: "2026-07-20T10:09:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:09:01Z" name: isvc-xgboost-graph-predictor-555fc69d56-ldfq2.18c3f79f8a50f2a1 namespace: kserve-ci-e2e-test resourceVersion: "15000" uid: 0fb8e296-e175-4066-b218-42830d0fc552 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-245.ec2.internal source: component: kubelet host: ip-10-0-133-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ldfq2 namespace: kserve-ci-e2e-test resourceVersion: "14919" uid: 195ca154-8722-4be5-bf3f-faafda88d097 kind: Event lastTimestamp: "2026-07-20T10:09:02Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-20T10:09:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:09:02Z" name: isvc-xgboost-graph-predictor-555fc69d56-ldfq2.18c3f79f8d0105a2 namespace: kserve-ci-e2e-test resourceVersion: "15003" uid: 2e07b932-f1fd-4ecc-96e5-611490f0e658 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-245.ec2.internal source: component: kubelet host: ip-10-0-133-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ldfq2 namespace: kserve-ci-e2e-test resourceVersion: "14919" uid: 195ca154-8722-4be5-bf3f-faafda88d097 kind: Event lastTimestamp: "2026-07-20T10:09:02Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-20T10:09:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:09:02Z" name: isvc-xgboost-graph-predictor-555fc69d56-ldfq2.18c3f79f8d7c7326 namespace: kserve-ci-e2e-test resourceVersion: "15004" uid: b6c3cdd9-51ad-48a4-a942-d062b13f8343 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-245.ec2.internal source: component: kubelet host: ip-10-0-133-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ldfq2 namespace: kserve-ci-e2e-test resourceVersion: "14919" uid: 195ca154-8722-4be5-bf3f-faafda88d097 kind: Event lastTimestamp: "2026-07-20T10:09:05Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-20T10:09:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:09:05Z" name: isvc-xgboost-graph-predictor-555fc69d56-ldfq2.18c3f7a0793dd163 namespace: kserve-ci-e2e-test resourceVersion: "15063" uid: bb03bc18-abfd-4f09-850e-4a549de0fd53 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-245.ec2.internal source: component: kubelet host: ip-10-0-133-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ldfq2 namespace: kserve-ci-e2e-test resourceVersion: "14919" uid: 195ca154-8722-4be5-bf3f-faafda88d097 kind: Event lastTimestamp: "2026-07-20T10:09:30Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 24.21s (24.21s including waiting). Image size: 1306969339 bytes.' metadata: creationTimestamp: "2026-07-20T10:09:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:09:30Z" name: isvc-xgboost-graph-predictor-555fc69d56-ldfq2.18c3f7a61c518d87 namespace: kserve-ci-e2e-test resourceVersion: "15303" uid: 34ba7d37-a63d-42f8-a4ec-84782bf42cf6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-245.ec2.internal source: component: kubelet host: ip-10-0-133-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ldfq2 namespace: kserve-ci-e2e-test resourceVersion: "14919" uid: 195ca154-8722-4be5-bf3f-faafda88d097 kind: Event lastTimestamp: "2026-07-20T10:09:30Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-20T10:09:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:09:30Z" name: isvc-xgboost-graph-predictor-555fc69d56-ldfq2.18c3f7a620fe3451 namespace: kserve-ci-e2e-test resourceVersion: "15304" uid: 59d789df-f2c7-4e9f-b61b-46811bfd6231 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-245.ec2.internal source: component: kubelet host: ip-10-0-133-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ldfq2 namespace: kserve-ci-e2e-test resourceVersion: "14919" uid: 195ca154-8722-4be5-bf3f-faafda88d097 kind: Event lastTimestamp: "2026-07-20T10:09:30Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-20T10:09:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:09:30Z" name: isvc-xgboost-graph-predictor-555fc69d56-ldfq2.18c3f7a621b279dd namespace: kserve-ci-e2e-test resourceVersion: "15305" uid: 5f789774-da2c-4a00-9dcf-476377fcf9c4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-245.ec2.internal source: component: kubelet host: ip-10-0-133-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ldfq2 namespace: kserve-ci-e2e-test resourceVersion: "14919" uid: 195ca154-8722-4be5-bf3f-faafda88d097 kind: Event lastTimestamp: "2026-07-20T10:09:30Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-20T10:09:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:09:30Z" name: isvc-xgboost-graph-predictor-555fc69d56-ldfq2.18c3f7a621c07bfe namespace: kserve-ci-e2e-test resourceVersion: "15306" uid: 72197285-007f-4015-a2e1-0c56316a82b6 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-245.ec2.internal source: component: kubelet host: ip-10-0-133-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ldfq2 namespace: kserve-ci-e2e-test resourceVersion: "14919" uid: 195ca154-8722-4be5-bf3f-faafda88d097 kind: Event lastTimestamp: "2026-07-20T10:09:33Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.834s (2.834s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-20T10:09:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:09:33Z" name: isvc-xgboost-graph-predictor-555fc69d56-ldfq2.18c3f7a6caad90ee namespace: kserve-ci-e2e-test resourceVersion: "15344" uid: 646cabb7-f28f-4633-85d4-c86af7b8459a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-245.ec2.internal source: component: kubelet host: ip-10-0-133-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ldfq2 namespace: kserve-ci-e2e-test resourceVersion: "14919" uid: 195ca154-8722-4be5-bf3f-faafda88d097 kind: Event lastTimestamp: "2026-07-20T10:09:33Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-20T10:09:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:09:33Z" name: isvc-xgboost-graph-predictor-555fc69d56-ldfq2.18c3f7a6cf82c85c namespace: kserve-ci-e2e-test resourceVersion: "15349" uid: 89d326b5-c4cb-4bd0-8808-b46141fca7ac reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-245.ec2.internal source: component: kubelet host: ip-10-0-133-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ldfq2 namespace: kserve-ci-e2e-test resourceVersion: "14919" uid: 195ca154-8722-4be5-bf3f-faafda88d097 kind: Event lastTimestamp: "2026-07-20T10:09:33Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:09:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:09:33Z" name: isvc-xgboost-graph-predictor-555fc69d56-ldfq2.18c3f7a6d0edaa5e namespace: kserve-ci-e2e-test resourceVersion: "15350" uid: 3842677e-a380-42a4-a41f-e4411b5c2d19 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-245.ec2.internal source: component: kubelet host: ip-10-0-133-245.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-20T10:09:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ldfq2 namespace: kserve-ci-e2e-test resourceVersion: "14919" uid: 195ca154-8722-4be5-bf3f-faafda88d097 kind: Event lastTimestamp: "2026-07-20T10:10:21Z" message: 'Readiness probe failed: dial tcp 10.132.0.17:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-20T10:09:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:10:21Z" name: isvc-xgboost-graph-predictor-555fc69d56-ldfq2.18c3f7a743023c09 namespace: kserve-ci-e2e-test resourceVersion: "15768" uid: 73a3cd68-7b95-438a-aaf6-1d5bd93d8fd5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-245.ec2.internal source: component: kubelet host: ip-10-0-133-245.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ldfq2 namespace: kserve-ci-e2e-test resourceVersion: "14919" uid: 195ca154-8722-4be5-bf3f-faafda88d097 kind: Event lastTimestamp: "2026-07-20T10:11:21Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-20T10:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:11:21Z" name: isvc-xgboost-graph-predictor-555fc69d56-ldfq2.18c3f7c0056b9b48 namespace: kserve-ci-e2e-test resourceVersion: "16579" uid: e36a6ebb-2cd7-4ea9-8558-7cbd33995b1b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-245.ec2.internal source: component: kubelet host: ip-10-0-133-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-ldfq2 namespace: kserve-ci-e2e-test resourceVersion: "14919" uid: 195ca154-8722-4be5-bf3f-faafda88d097 kind: Event lastTimestamp: "2026-07-20T10:11:21Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:11:21Z" name: isvc-xgboost-graph-predictor-555fc69d56-ldfq2.18c3f7c0056c2557 namespace: kserve-ci-e2e-test resourceVersion: "16581" uid: bb860684-5909-46e6-a15e-b358c5ab6797 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-245.ec2.internal source: component: kubelet host: ip-10-0-133-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-555fc69d56 namespace: kserve-ci-e2e-test resourceVersion: "14909" uid: f85903a0-b4f7-4e6b-8bd5-e1c66207acfc kind: Event lastTimestamp: "2026-07-20T10:09:01Z" message: 'Created pod: isvc-xgboost-graph-predictor-555fc69d56-ldfq2' metadata: creationTimestamp: "2026-07-20T10:09:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:09:01Z" name: isvc-xgboost-graph-predictor-555fc69d56.18c3f79f6ed0c5c9 namespace: kserve-ci-e2e-test resourceVersion: "14918" uid: d86ae58e-bed5-4ff4-9072-e0896ab106eb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14907" uid: db0a6c0e-d29a-4ffa-959d-1ecaf4122723 kind: Event lastTimestamp: "2026-07-20T10:09:01Z" message: Scaled up replica set isvc-xgboost-graph-predictor-555fc69d56 from 0 to 1 metadata: creationTimestamp: "2026-07-20T10:09:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:09:01Z" name: isvc-xgboost-graph-predictor.18c3f79f6bff9d1f namespace: kserve-ci-e2e-test resourceVersion: "14910" uid: 53d9f3da-e471-41f4-ba69-14332756004a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-20T10:09:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14908" uid: 77561c92-66cf-47a1-bbd8-26cd652a540e kind: Event lastTimestamp: "2026-07-20T10:10:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:09:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:10:01Z" name: isvc-xgboost-graph-predictor.18c3f7a2ea65983d namespace: kserve-ci-e2e-test resourceVersion: "15577" uid: 400ff75d-6260-435d-9924-11983c1460cb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-20T10:09:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14908" uid: 77561c92-66cf-47a1-bbd8-26cd652a540e kind: Event lastTimestamp: "2026-07-20T10:10:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:09:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:10:01Z" name: isvc-xgboost-graph-predictor.18c3f7a2eace8653 namespace: kserve-ci-e2e-test resourceVersion: "15578" uid: cbb18a78-3cee-4b91-afdf-b55852a21569 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-20T10:10:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15151" uid: 77561c92-66cf-47a1-bbd8-26cd652a540e kind: Event lastTimestamp: "2026-07-20T10:11:01Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:10:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:11:01Z" name: isvc-xgboost-graph-predictor.18c3f7b0e6e4b9b6 namespace: kserve-ci-e2e-test resourceVersion: "16285" uid: 95d0058a-0178-4635-942a-32840afbf18f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-20T10:10:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15151" uid: 77561c92-66cf-47a1-bbd8-26cd652a540e kind: Event lastTimestamp: "2026-07-20T10:11:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:10:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:11:01Z" name: isvc-xgboost-graph-predictor.18c3f7b0e7435a24 namespace: kserve-ci-e2e-test resourceVersion: "16287" uid: 1feaa1f1-1c92-495e-aadc-88a453c5f758 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-20T10:09:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14893" uid: aae0efd1-d3da-4e76-9633-c1fbaa724048 kind: Event lastTimestamp: "2026-07-20T10:10:31Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-20T10:09:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:10:31Z" name: isvc-xgboost-graph.18c3f79f6c01c136 namespace: kserve-ci-e2e-test resourceVersion: "15905" uid: 2fdfeb1e-fa3e-47a8-9ad1-dfea385eb9fe reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15888" uid: aae0efd1-d3da-4e76-9633-c1fbaa724048 kind: Event lastTimestamp: "2026-07-20T10:10:31Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-07-20T10:10:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:10:31Z" name: isvc-xgboost-graph.18c3f7b454c043cb namespace: kserve-ci-e2e-test resourceVersion: "15889" uid: fbfa6205-8d24-4614-ae1a-1dbd39453ee3 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-20T10:11:11.094314Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-8df46b748-9ll4c namespace: kserve-ci-e2e-test resourceVersion: "16365" uid: b306f8d8-3900-4432-bbab-b8be3191e22d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-8df46b748-9ll4c to ip-10-0-138-143.ec2.internal metadata: creationTimestamp: "2026-07-20T10:11:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-20T10:11:11Z" name: model-chainer-8df46b748-9ll4c.18c3f7bd9b3ee7ac namespace: kserve-ci-e2e-test resourceVersion: "16370" uid: ad33825a-9e70-4d2c-9e2a-5b391fdd820d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5dc669d9-69hkz source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-20T10:11:11Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-8df46b748-9ll4c namespace: kserve-ci-e2e-test resourceVersion: "16369" uid: b306f8d8-3900-4432-bbab-b8be3191e22d kind: Event lastTimestamp: "2026-07-20T10:11:11Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-serving-cert" not found' metadata: creationTimestamp: "2026-07-20T10:11:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:11:11Z" name: model-chainer-8df46b748-9ll4c.18c3f7bda87fc05a namespace: kserve-ci-e2e-test resourceVersion: "16387" uid: e26487d1-3189-4a8c-99c5-b900d585fd05 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:13Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-8df46b748-9ll4c namespace: kserve-ci-e2e-test resourceVersion: "16374" uid: b306f8d8-3900-4432-bbab-b8be3191e22d kind: Event lastTimestamp: "2026-07-20T10:11:13Z" message: Add eth0 [10.134.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-20T10:11:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-20T10:11:13Z" name: model-chainer-8df46b748-9ll4c.18c3f7be0e0f16cc namespace: kserve-ci-e2e-test resourceVersion: "16398" uid: db10679a-f979-4e98-8f04-3cdca199141f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-8df46b748-9ll4c namespace: kserve-ci-e2e-test resourceVersion: "16369" uid: b306f8d8-3900-4432-bbab-b8be3191e22d kind: Event lastTimestamp: "2026-07-20T10:11:13Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:ee73d7a22df8b79aa8a5e917cb435b1d2b100d4582439c437bdc01a1d2422290" already present on machine metadata: creationTimestamp: "2026-07-20T10:11:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:11:13Z" name: model-chainer-8df46b748-9ll4c.18c3f7be0fb0b31d namespace: kserve-ci-e2e-test resourceVersion: "16400" uid: 01643330-7bc8-4a0a-bd06-14cd42fd967e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-8df46b748-9ll4c namespace: kserve-ci-e2e-test resourceVersion: "16369" uid: b306f8d8-3900-4432-bbab-b8be3191e22d kind: Event lastTimestamp: "2026-07-20T10:11:13Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-07-20T10:11:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:11:13Z" name: model-chainer-8df46b748-9ll4c.18c3f7be12cae72a namespace: kserve-ci-e2e-test resourceVersion: "16401" uid: 710aac8c-d55f-4baf-84c6-f06bd35a9b2c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-8df46b748-9ll4c namespace: kserve-ci-e2e-test resourceVersion: "16369" uid: b306f8d8-3900-4432-bbab-b8be3191e22d kind: Event lastTimestamp: "2026-07-20T10:11:13Z" message: Started container model-chainer metadata: creationTimestamp: "2026-07-20T10:11:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:11:13Z" name: model-chainer-8df46b748-9ll4c.18c3f7be13378f04 namespace: kserve-ci-e2e-test resourceVersion: "16402" uid: 966d91b4-74fa-4226-8fda-8d62b45695f4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-8df46b748-9ll4c namespace: kserve-ci-e2e-test resourceVersion: "16369" uid: b306f8d8-3900-4432-bbab-b8be3191e22d kind: Event lastTimestamp: "2026-07-20T10:11:21Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-07-20T10:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:11:21Z" name: model-chainer-8df46b748-9ll4c.18c3f7bff463b7f5 namespace: kserve-ci-e2e-test resourceVersion: "16514" uid: 19cb199c-5f2f-42bb-86e9-c2295f82327a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-20T10:11:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-8df46b748-9ll4c namespace: kserve-ci-e2e-test resourceVersion: "16369" uid: b306f8d8-3900-4432-bbab-b8be3191e22d kind: Event lastTimestamp: "2026-07-20T10:11:49Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-20T10:11:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:11:49Z" name: model-chainer-8df46b748-9ll4c.18c3f7c0cfa23f3d namespace: kserve-ci-e2e-test resourceVersion: "16937" uid: 654d2a56-5293-4b0b-b6e3-32527bb60a9b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-8df46b748 namespace: kserve-ci-e2e-test resourceVersion: "16362" uid: 862f4c2d-8e88-4486-9628-1c9890a7e454 kind: Event lastTimestamp: "2026-07-20T10:11:11Z" message: 'Created pod: model-chainer-8df46b748-9ll4c' metadata: creationTimestamp: "2026-07-20T10:11:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:11:11Z" name: model-chainer-8df46b748.18c3f7bd9a85131a namespace: kserve-ci-e2e-test resourceVersion: "16367" uid: 21304d27-cf3b-450a-b492-19f5df14f80c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16360" uid: cee4ee83-2656-412e-ac69-3787fa1252fe kind: Event lastTimestamp: "2026-07-20T10:11:11Z" message: Scaled up replica set model-chainer-8df46b748 from 0 to 1 metadata: creationTimestamp: "2026-07-20T10:11:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:11:11Z" name: model-chainer.18c3f7bd992f4d70 namespace: kserve-ci-e2e-test resourceVersion: "16363" uid: abcb19d9-78f2-465f-80f7-26551f6a443a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:11Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16376" uid: 209d2cc8-fd8e-4502-97ff-7c6f8d978651 kind: Event lastTimestamp: "2026-07-20T10:11:11Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-07-20T10:11:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:11:11Z" name: model-chainer.18c3f7bd9f465759 namespace: kserve-ci-e2e-test resourceVersion: "16378" uid: 6405d706-2a2a-406c-95bf-5bc364ff7ab7 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-20T10:21:46.266753Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-24b48-8867c854c-crmlj namespace: kserve-ci-e2e-test resourceVersion: "21767" uid: de936678-cc94-44d5-ab32-1315bd54b2f1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-24b48-8867c854c-crmlj to ip-10-0-138-143.ec2.internal metadata: creationTimestamp: "2026-07-20T10:21:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-20T10:21:46Z" name: sequence-graph-24b48-8867c854c-crmlj.18c3f8517e792f4b namespace: kserve-ci-e2e-test resourceVersion: "21772" uid: 56f6197c-ee8a-4799-8623-ac5b22dad3e6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5dc669d9-69hkz source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:46Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-24b48-8867c854c-crmlj namespace: kserve-ci-e2e-test resourceVersion: "21775" uid: de936678-cc94-44d5-ab32-1315bd54b2f1 kind: Event lastTimestamp: "2026-07-20T10:21:46Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-20T10:21:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-20T10:21:46Z" name: sequence-graph-24b48-8867c854c-crmlj.18c3f85197db4728 namespace: kserve-ci-e2e-test resourceVersion: "21786" uid: a29ed39f-18d1-479e-a9b6-4e5392a3e12c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-24b48} kind: Pod name: sequence-graph-24b48-8867c854c-crmlj namespace: kserve-ci-e2e-test resourceVersion: "21768" uid: de936678-cc94-44d5-ab32-1315bd54b2f1 kind: Event lastTimestamp: "2026-07-20T10:21:46Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:ee73d7a22df8b79aa8a5e917cb435b1d2b100d4582439c437bdc01a1d2422290" already present on machine metadata: creationTimestamp: "2026-07-20T10:21:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:21:46Z" name: sequence-graph-24b48-8867c854c-crmlj.18c3f85199444c15 namespace: kserve-ci-e2e-test resourceVersion: "21788" uid: 8afdf8b2-38d4-4257-9cf1-9cea090702c4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-24b48} kind: Pod name: sequence-graph-24b48-8867c854c-crmlj namespace: kserve-ci-e2e-test resourceVersion: "21768" uid: de936678-cc94-44d5-ab32-1315bd54b2f1 kind: Event lastTimestamp: "2026-07-20T10:21:46Z" message: 'Created container: sequence-graph-24b48' metadata: creationTimestamp: "2026-07-20T10:21:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:21:46Z" name: sequence-graph-24b48-8867c854c-crmlj.18c3f8519c438cd3 namespace: kserve-ci-e2e-test resourceVersion: "21789" uid: fb8672da-0661-4b8c-bfc4-682952dd5918 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-24b48} kind: Pod name: sequence-graph-24b48-8867c854c-crmlj namespace: kserve-ci-e2e-test resourceVersion: "21768" uid: de936678-cc94-44d5-ab32-1315bd54b2f1 kind: Event lastTimestamp: "2026-07-20T10:21:46Z" message: Started container sequence-graph-24b48 metadata: creationTimestamp: "2026-07-20T10:21:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:21:46Z" name: sequence-graph-24b48-8867c854c-crmlj.18c3f8519cb2a367 namespace: kserve-ci-e2e-test resourceVersion: "21790" uid: 6c7ee011-ebec-44eb-8e55-a65719ca3b46 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-24b48} kind: Pod name: sequence-graph-24b48-8867c854c-crmlj namespace: kserve-ci-e2e-test resourceVersion: "21768" uid: de936678-cc94-44d5-ab32-1315bd54b2f1 kind: Event lastTimestamp: "2026-07-20T10:21:56Z" message: Stopping container sequence-graph-24b48 metadata: creationTimestamp: "2026-07-20T10:21:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:21:56Z" name: sequence-graph-24b48-8867c854c-crmlj.18c3f853d6251d7e namespace: kserve-ci-e2e-test resourceVersion: "21880" uid: 9350233f-1d13-4bc1-8117-cd9aa30cb4a1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-20T10:21:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-24b48} kind: Pod name: sequence-graph-24b48-8867c854c-crmlj namespace: kserve-ci-e2e-test resourceVersion: "21768" uid: de936678-cc94-44d5-ab32-1315bd54b2f1 kind: Event lastTimestamp: "2026-07-20T10:22:23Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-20T10:21:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:22:23Z" name: sequence-graph-24b48-8867c854c-crmlj.18c3f854484ddf01 namespace: kserve-ci-e2e-test resourceVersion: "22269" uid: f45a84f2-6dfb-4142-8e84-335409f25cb6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:46Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-24b48-8867c854c namespace: kserve-ci-e2e-test resourceVersion: "21764" uid: 27ed1a66-9303-4dd0-845e-074b5944a915 kind: Event lastTimestamp: "2026-07-20T10:21:46Z" message: 'Created pod: sequence-graph-24b48-8867c854c-crmlj' metadata: creationTimestamp: "2026-07-20T10:21:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:21:46Z" name: sequence-graph-24b48-8867c854c.18c3f8517de2a3c0 namespace: kserve-ci-e2e-test resourceVersion: "21769" uid: e8467501-9821-4cce-adeb-78c737a2e2fe reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:46Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-24b48 namespace: kserve-ci-e2e-test resourceVersion: "21762" uid: fbe6f85d-b5d0-4ea7-81f8-73c122a864e6 kind: Event lastTimestamp: "2026-07-20T10:21:46Z" message: Scaled up replica set sequence-graph-24b48-8867c854c from 0 to 1 metadata: creationTimestamp: "2026-07-20T10:21:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:21:46Z" name: sequence-graph-24b48.18c3f8517d058014 namespace: kserve-ci-e2e-test resourceVersion: "21765" uid: c8e45556-026e-4222-ac0e-19678cda86a1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:46Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-24b48 namespace: kserve-ci-e2e-test resourceVersion: "21779" uid: 1da0dfcc-082b-4b5d-9c60-2de82d48e781 kind: Event lastTimestamp: "2026-07-20T10:21:46Z" message: InferenceGraph [sequence-graph-24b48] is Ready metadata: creationTimestamp: "2026-07-20T10:21:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:21:46Z" name: sequence-graph-24b48.18c3f8518361f457 namespace: kserve-ci-e2e-test resourceVersion: "21781" uid: f24742d7-84e5-49a6-889d-f305606d98fc reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-20T10:22:56.619216Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-64e78-fd7466b96-mj96x namespace: kserve-ci-e2e-test resourceVersion: "22551" uid: caaff770-02cb-454d-954b-f960bfc3638c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-64e78-fd7466b96-mj96x to ip-10-0-138-143.ec2.internal metadata: creationTimestamp: "2026-07-20T10:22:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-20T10:22:56Z" name: sequence-graph-64e78-fd7466b96-mj96x.18c3f861dfce948d namespace: kserve-ci-e2e-test resourceVersion: "22556" uid: 16b50e2a-b3a9-4117-a944-b65c3ca67677 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5dc669d9-69hkz source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:22:56Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-64e78-fd7466b96-mj96x namespace: kserve-ci-e2e-test resourceVersion: "22552" uid: caaff770-02cb-454d-954b-f960bfc3638c kind: Event lastTimestamp: "2026-07-20T10:22:56Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-64e78-serving-cert" not found' metadata: creationTimestamp: "2026-07-20T10:22:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:22:56Z" name: sequence-graph-64e78-fd7466b96-mj96x.18c3f861ed71d30b namespace: kserve-ci-e2e-test resourceVersion: "22573" uid: 2725f2ce-a711-4dbb-9108-0ff191b20b94 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:22:57Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-64e78-fd7466b96-mj96x namespace: kserve-ci-e2e-test resourceVersion: "22560" uid: caaff770-02cb-454d-954b-f960bfc3638c kind: Event lastTimestamp: "2026-07-20T10:22:57Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-20T10:22:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-20T10:22:57Z" name: sequence-graph-64e78-fd7466b96-mj96x.18c3f8621d0d4e34 namespace: kserve-ci-e2e-test resourceVersion: "22585" uid: ddf4278d-de15-4a7a-abdb-0722a37212a6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:22:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-64e78} kind: Pod name: sequence-graph-64e78-fd7466b96-mj96x namespace: kserve-ci-e2e-test resourceVersion: "22552" uid: caaff770-02cb-454d-954b-f960bfc3638c kind: Event lastTimestamp: "2026-07-20T10:22:57Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:ee73d7a22df8b79aa8a5e917cb435b1d2b100d4582439c437bdc01a1d2422290" already present on machine metadata: creationTimestamp: "2026-07-20T10:22:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:22:57Z" name: sequence-graph-64e78-fd7466b96-mj96x.18c3f8621e7018ed namespace: kserve-ci-e2e-test resourceVersion: "22587" uid: e10a809d-30c3-4faa-9663-e54fbeca2668 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:22:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-64e78} kind: Pod name: sequence-graph-64e78-fd7466b96-mj96x namespace: kserve-ci-e2e-test resourceVersion: "22552" uid: caaff770-02cb-454d-954b-f960bfc3638c kind: Event lastTimestamp: "2026-07-20T10:22:57Z" message: 'Created container: sequence-graph-64e78' metadata: creationTimestamp: "2026-07-20T10:22:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:22:57Z" name: sequence-graph-64e78-fd7466b96-mj96x.18c3f862211bbf4c namespace: kserve-ci-e2e-test resourceVersion: "22588" uid: 20245de5-a95b-4641-a628-b73023117f1d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:22:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-64e78} kind: Pod name: sequence-graph-64e78-fd7466b96-mj96x namespace: kserve-ci-e2e-test resourceVersion: "22552" uid: caaff770-02cb-454d-954b-f960bfc3638c kind: Event lastTimestamp: "2026-07-20T10:22:57Z" message: Started container sequence-graph-64e78 metadata: creationTimestamp: "2026-07-20T10:22:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:22:57Z" name: sequence-graph-64e78-fd7466b96-mj96x.18c3f86221e1fe89 namespace: kserve-ci-e2e-test resourceVersion: "22589" uid: d19b94fc-e1f6-4cc8-a752-e5a6c15c2409 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-64e78} kind: Pod name: sequence-graph-64e78-fd7466b96-mj96x namespace: kserve-ci-e2e-test resourceVersion: "22552" uid: caaff770-02cb-454d-954b-f960bfc3638c kind: Event lastTimestamp: "2026-07-20T10:31:11Z" message: Stopping container sequence-graph-64e78 metadata: creationTimestamp: "2026-07-20T10:31:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:31:11Z" name: sequence-graph-64e78-fd7466b96-mj96x.18c3f8d503b6045a namespace: kserve-ci-e2e-test resourceVersion: "26493" uid: 8f596948-bf7c-445d-91da-334aa2af6222 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-20T10:31:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-64e78} kind: Pod name: sequence-graph-64e78-fd7466b96-mj96x namespace: kserve-ci-e2e-test resourceVersion: "22552" uid: caaff770-02cb-454d-954b-f960bfc3638c kind: Event lastTimestamp: "2026-07-20T10:31:39Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-20T10:31:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:31:39Z" name: sequence-graph-64e78-fd7466b96-mj96x.18c3f8d5cd01ab9b namespace: kserve-ci-e2e-test resourceVersion: "27055" uid: 8d25e879-4607-4f9f-a5d7-6399de1782ba reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:22:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-64e78-fd7466b96 namespace: kserve-ci-e2e-test resourceVersion: "22548" uid: e7122dd2-3403-45c4-a296-ba343e5c8b53 kind: Event lastTimestamp: "2026-07-20T10:22:56Z" message: 'Created pod: sequence-graph-64e78-fd7466b96-mj96x' metadata: creationTimestamp: "2026-07-20T10:22:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:22:56Z" name: sequence-graph-64e78-fd7466b96.18c3f861df267eea namespace: kserve-ci-e2e-test resourceVersion: "22553" uid: 9bc71682-9377-4853-8775-833675205caf reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:22:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-64e78 namespace: kserve-ci-e2e-test resourceVersion: "22545" uid: b4a71255-5a8d-486f-9d57-47720b84771d kind: Event lastTimestamp: "2026-07-20T10:22:56Z" message: Scaled up replica set sequence-graph-64e78-fd7466b96 from 0 to 1 metadata: creationTimestamp: "2026-07-20T10:22:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:22:56Z" name: sequence-graph-64e78.18c3f861de21db18 namespace: kserve-ci-e2e-test resourceVersion: "22549" uid: a3d56e86-8d70-4a9f-990e-aeb11888506c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:22:56Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-64e78 namespace: kserve-ci-e2e-test resourceVersion: "22565" uid: b5e6b6d3-09f4-49ac-9284-7f82b1e759bd kind: Event lastTimestamp: "2026-07-20T10:22:56Z" message: InferenceGraph [sequence-graph-64e78] is Ready metadata: creationTimestamp: "2026-07-20T10:22:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:22:56Z" name: sequence-graph-64e78.18c3f861e5614c1b namespace: kserve-ci-e2e-test resourceVersion: "22567" uid: 8810b373-719a-49fc-8d03-af321c41fb91 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:23:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-64e78 namespace: kserve-ci-e2e-test resourceVersion: "22547" uid: c2ed0d8e-8818-4f22-ab63-b38dd763c869 kind: Event lastTimestamp: "2026-07-20T10:23:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:23:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:23:11Z" name: sequence-graph-64e78.18c3f8655cd89b19 namespace: kserve-ci-e2e-test resourceVersion: "22691" uid: 1c17797e-562f-4697-8b8d-b25cb8ea103b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:23:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-64e78 namespace: kserve-ci-e2e-test resourceVersion: "22547" uid: c2ed0d8e-8818-4f22-ab63-b38dd763c869 kind: Event lastTimestamp: "2026-07-20T10:23:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:23:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:23:11Z" name: sequence-graph-64e78.18c3f8655d558c3b namespace: kserve-ci-e2e-test resourceVersion: "22692" uid: e2140ab5-f6fc-4336-abdb-b8accd3cb74b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:23:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-64e78 namespace: kserve-ci-e2e-test resourceVersion: "22690" uid: c2ed0d8e-8818-4f22-ab63-b38dd763c869 kind: Event lastTimestamp: "2026-07-20T10:23:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:23:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:23:26Z" name: sequence-graph-64e78.18c3f868dbf9b3bd namespace: kserve-ci-e2e-test resourceVersion: "22804" uid: f1095abb-1033-48c8-82db-0236a2c57208 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:23:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-64e78 namespace: kserve-ci-e2e-test resourceVersion: "22690" uid: c2ed0d8e-8818-4f22-ab63-b38dd763c869 kind: Event lastTimestamp: "2026-07-20T10:23:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:23:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:23:26Z" name: sequence-graph-64e78.18c3f868dc64cc8f namespace: kserve-ci-e2e-test resourceVersion: "22806" uid: 1fcea537-089d-4bbc-9f97-58369153612e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-20T10:12:21.421723Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-78f99-5f589d8f54-4zpv9 namespace: kserve-ci-e2e-test resourceVersion: "17191" uid: b852594b-cab0-48ff-98d1-48993d9ff6a5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-78f99-5f589d8f54-4zpv9 to ip-10-0-138-143.ec2.internal metadata: creationTimestamp: "2026-07-20T10:12:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-20T10:12:21Z" name: sequence-graph-78f99-5f589d8f54-4zpv9.18c3f7cdfb1604e0 namespace: kserve-ci-e2e-test resourceVersion: "17196" uid: e54ec60c-9e63-46a9-8dff-fccdef3b9c06 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5dc669d9-69hkz source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:12:21Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-78f99-5f589d8f54-4zpv9 namespace: kserve-ci-e2e-test resourceVersion: "17201" uid: b852594b-cab0-48ff-98d1-48993d9ff6a5 kind: Event lastTimestamp: "2026-07-20T10:12:21Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-20T10:12:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-20T10:12:21Z" name: sequence-graph-78f99-5f589d8f54-4zpv9.18c3f7ce14d06dbe namespace: kserve-ci-e2e-test resourceVersion: "17217" uid: 8a4db635-f13d-43dc-8e79-eadfb631db8a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:12:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-78f99} kind: Pod name: sequence-graph-78f99-5f589d8f54-4zpv9 namespace: kserve-ci-e2e-test resourceVersion: "17193" uid: b852594b-cab0-48ff-98d1-48993d9ff6a5 kind: Event lastTimestamp: "2026-07-20T10:12:21Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:ee73d7a22df8b79aa8a5e917cb435b1d2b100d4582439c437bdc01a1d2422290" already present on machine metadata: creationTimestamp: "2026-07-20T10:12:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:12:21Z" name: sequence-graph-78f99-5f589d8f54-4zpv9.18c3f7ce16312f1d namespace: kserve-ci-e2e-test resourceVersion: "17219" uid: 74e3ae22-eb1e-4dfe-9f21-344fec9c2fc9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:12:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-78f99} kind: Pod name: sequence-graph-78f99-5f589d8f54-4zpv9 namespace: kserve-ci-e2e-test resourceVersion: "17193" uid: b852594b-cab0-48ff-98d1-48993d9ff6a5 kind: Event lastTimestamp: "2026-07-20T10:12:21Z" message: 'Created container: sequence-graph-78f99' metadata: creationTimestamp: "2026-07-20T10:12:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:12:21Z" name: sequence-graph-78f99-5f589d8f54-4zpv9.18c3f7ce1943e67a namespace: kserve-ci-e2e-test resourceVersion: "17220" uid: 7307abc0-008c-48e6-acd6-8478dcc7f13e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:12:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-78f99} kind: Pod name: sequence-graph-78f99-5f589d8f54-4zpv9 namespace: kserve-ci-e2e-test resourceVersion: "17193" uid: b852594b-cab0-48ff-98d1-48993d9ff6a5 kind: Event lastTimestamp: "2026-07-20T10:12:21Z" message: Started container sequence-graph-78f99 metadata: creationTimestamp: "2026-07-20T10:12:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:12:21Z" name: sequence-graph-78f99-5f589d8f54-4zpv9.18c3f7ce19b7fffe namespace: kserve-ci-e2e-test resourceVersion: "17221" uid: 7a3ef035-bd68-4661-a793-fb3031b79d9b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-78f99} kind: Pod name: sequence-graph-78f99-5f589d8f54-4zpv9 namespace: kserve-ci-e2e-test resourceVersion: "17193" uid: b852594b-cab0-48ff-98d1-48993d9ff6a5 kind: Event lastTimestamp: "2026-07-20T10:20:36Z" message: Stopping container sequence-graph-78f99 metadata: creationTimestamp: "2026-07-20T10:20:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:20:36Z" name: sequence-graph-78f99-5f589d8f54-4zpv9.18c3f841249c91b7 namespace: kserve-ci-e2e-test resourceVersion: "20832" uid: 94c7a1e1-7306-4845-a1c1-905b938eea94 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-20T10:20:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-78f99} kind: Pod name: sequence-graph-78f99-5f589d8f54-4zpv9 namespace: kserve-ci-e2e-test resourceVersion: "17193" uid: b852594b-cab0-48ff-98d1-48993d9ff6a5 kind: Event lastTimestamp: "2026-07-20T10:21:03Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-20T10:20:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:21:03Z" name: sequence-graph-78f99-5f589d8f54-4zpv9.18c3f841a0a487db namespace: kserve-ci-e2e-test resourceVersion: "21365" uid: 20c4b3e3-3c05-45ca-97c4-048a0133a975 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:12:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-78f99-5f589d8f54 namespace: kserve-ci-e2e-test resourceVersion: "17188" uid: 5fc69e70-95bb-433e-9fc3-e4d01930d5f2 kind: Event lastTimestamp: "2026-07-20T10:12:21Z" message: 'Created pod: sequence-graph-78f99-5f589d8f54-4zpv9' metadata: creationTimestamp: "2026-07-20T10:12:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:12:21Z" name: sequence-graph-78f99-5f589d8f54.18c3f7cdfaa38ca2 namespace: kserve-ci-e2e-test resourceVersion: "17192" uid: 821fd883-7e10-4322-abc0-fe785ddb7719 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:12:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-78f99 namespace: kserve-ci-e2e-test resourceVersion: "17186" uid: 00c4bcf6-877f-443a-b2b1-c99b2b313f84 kind: Event lastTimestamp: "2026-07-20T10:12:21Z" message: Scaled up replica set sequence-graph-78f99-5f589d8f54 from 0 to 1 metadata: creationTimestamp: "2026-07-20T10:12:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:12:21Z" name: sequence-graph-78f99.18c3f7cdf9c30892 namespace: kserve-ci-e2e-test resourceVersion: "17189" uid: 342b7f30-6c2c-46f8-9ca7-f7e59c819be4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:12:21Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-78f99 namespace: kserve-ci-e2e-test resourceVersion: "17204" uid: 30730c60-d03d-420f-bc41-979c453a5f50 kind: Event lastTimestamp: "2026-07-20T10:12:21Z" message: InferenceGraph [sequence-graph-78f99] is Ready metadata: creationTimestamp: "2026-07-20T10:12:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:12:21Z" name: sequence-graph-78f99.18c3f7cdff508e77 namespace: kserve-ci-e2e-test resourceVersion: "17206" uid: b09a959f-9741-4b08-95ab-fe6804a643c8 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:12:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-78f99 namespace: kserve-ci-e2e-test resourceVersion: "17187" uid: 5eb5ec1d-0298-4131-a91c-01013a1390a6 kind: Event lastTimestamp: "2026-07-20T10:12:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:12:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:12:36Z" name: sequence-graph-78f99.18c3f7d178d8e230 namespace: kserve-ci-e2e-test resourceVersion: "17329" uid: 1c8c4305-eeaa-4073-8408-8d0612283aa9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:12:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-78f99 namespace: kserve-ci-e2e-test resourceVersion: "17187" uid: 5eb5ec1d-0298-4131-a91c-01013a1390a6 kind: Event lastTimestamp: "2026-07-20T10:12:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:12:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:12:36Z" name: sequence-graph-78f99.18c3f7d1797f76cf namespace: kserve-ci-e2e-test resourceVersion: "17332" uid: 4c81ea88-21e0-41e7-ac23-6703036cfd32 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:12:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-78f99 namespace: kserve-ci-e2e-test resourceVersion: "17331" uid: 5eb5ec1d-0298-4131-a91c-01013a1390a6 kind: Event lastTimestamp: "2026-07-20T10:12:51Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:12:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:12:51Z" name: sequence-graph-78f99.18c3f7d4f836f99d namespace: kserve-ci-e2e-test resourceVersion: "17433" uid: fe5c742c-8e36-45be-98a6-8254970d4256 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:12:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-78f99 namespace: kserve-ci-e2e-test resourceVersion: "17331" uid: 5eb5ec1d-0298-4131-a91c-01013a1390a6 kind: Event lastTimestamp: "2026-07-20T10:12:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:12:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:12:51Z" name: sequence-graph-78f99.18c3f7d4f886cfb2 namespace: kserve-ci-e2e-test resourceVersion: "17434" uid: 4d61b1fd-2172-4915-b62e-142effdd7aa5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-20T10:32:32.219042Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-6e423-b7859bb99-xcxhd namespace: kserve-ci-e2e-test resourceVersion: "27556" uid: 8ffe3d4a-bf07-41a8-b3a0-cf24cea8b764 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-6e423-b7859bb99-xcxhd to ip-10-0-138-143.ec2.internal metadata: creationTimestamp: "2026-07-20T10:32:32Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-20T10:32:32Z" name: splitter-graph-6e423-b7859bb99-xcxhd.18c3f8e7e43afbe4 namespace: kserve-ci-e2e-test resourceVersion: "27562" uid: 1e4a92c0-7c3e-4efb-b7f4-8538ed7bc14a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5dc669d9-69hkz source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:32:32Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-6e423-b7859bb99-xcxhd namespace: kserve-ci-e2e-test resourceVersion: "27565" uid: 8ffe3d4a-bf07-41a8-b3a0-cf24cea8b764 kind: Event lastTimestamp: "2026-07-20T10:32:32Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-20T10:32:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-20T10:32:32Z" name: splitter-graph-6e423-b7859bb99-xcxhd.18c3f8e7fdcc94bc namespace: kserve-ci-e2e-test resourceVersion: "27575" uid: 6435e456-cd3d-4785-9910-17a19e76cff7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:32:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-6e423} kind: Pod name: splitter-graph-6e423-b7859bb99-xcxhd namespace: kserve-ci-e2e-test resourceVersion: "27559" uid: 8ffe3d4a-bf07-41a8-b3a0-cf24cea8b764 kind: Event lastTimestamp: "2026-07-20T10:32:32Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:ee73d7a22df8b79aa8a5e917cb435b1d2b100d4582439c437bdc01a1d2422290" already present on machine metadata: creationTimestamp: "2026-07-20T10:32:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:32:32Z" name: splitter-graph-6e423-b7859bb99-xcxhd.18c3f8e7ff009c20 namespace: kserve-ci-e2e-test resourceVersion: "27577" uid: 803e4a36-6afa-4499-a6f2-4ee981384f64 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:32:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-6e423} kind: Pod name: splitter-graph-6e423-b7859bb99-xcxhd namespace: kserve-ci-e2e-test resourceVersion: "27559" uid: 8ffe3d4a-bf07-41a8-b3a0-cf24cea8b764 kind: Event lastTimestamp: "2026-07-20T10:32:32Z" message: 'Created container: splitter-graph-6e423' metadata: creationTimestamp: "2026-07-20T10:32:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:32:32Z" name: splitter-graph-6e423-b7859bb99-xcxhd.18c3f8e801c1de9a namespace: kserve-ci-e2e-test resourceVersion: "27578" uid: ccf8a7a3-b6cb-427a-bc2a-8e13ecc70e76 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:32:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-6e423} kind: Pod name: splitter-graph-6e423-b7859bb99-xcxhd namespace: kserve-ci-e2e-test resourceVersion: "27559" uid: 8ffe3d4a-bf07-41a8-b3a0-cf24cea8b764 kind: Event lastTimestamp: "2026-07-20T10:32:32Z" message: Started container splitter-graph-6e423 metadata: creationTimestamp: "2026-07-20T10:32:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:32:32Z" name: splitter-graph-6e423-b7859bb99-xcxhd.18c3f8e8022df7c4 namespace: kserve-ci-e2e-test resourceVersion: "27579" uid: 291cff5b-2fbf-4206-b836-5f91b31c4519 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:40:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-6e423} kind: Pod name: splitter-graph-6e423-b7859bb99-xcxhd namespace: kserve-ci-e2e-test resourceVersion: "27559" uid: 8ffe3d4a-bf07-41a8-b3a0-cf24cea8b764 kind: Event lastTimestamp: "2026-07-20T10:40:46Z" message: Stopping container splitter-graph-6e423 metadata: creationTimestamp: "2026-07-20T10:40:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:40:46Z" name: splitter-graph-6e423-b7859bb99-xcxhd.18c3f95b08a6269c namespace: kserve-ci-e2e-test resourceVersion: "31024" uid: e6adc5de-a281-48a6-be56-1878e02b86c4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-20T10:40:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-6e423} kind: Pod name: splitter-graph-6e423-b7859bb99-xcxhd namespace: kserve-ci-e2e-test resourceVersion: "27559" uid: 8ffe3d4a-bf07-41a8-b3a0-cf24cea8b764 kind: Event lastTimestamp: "2026-07-20T10:41:14Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-20T10:40:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:41:14Z" name: splitter-graph-6e423-b7859bb99-xcxhd.18c3f95bb87dccd1 namespace: kserve-ci-e2e-test resourceVersion: "31265" uid: 35512774-3981-4d85-a6f7-446ce9fdc8a1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:32:32Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-6e423-b7859bb99 namespace: kserve-ci-e2e-test resourceVersion: "27553" uid: 49c64b07-d086-4ab7-8ba9-a7e17f91c5fa kind: Event lastTimestamp: "2026-07-20T10:32:32Z" message: 'Created pod: splitter-graph-6e423-b7859bb99-xcxhd' metadata: creationTimestamp: "2026-07-20T10:32:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:32:32Z" name: splitter-graph-6e423-b7859bb99.18c3f8e7e3416aac namespace: kserve-ci-e2e-test resourceVersion: "27557" uid: ed9abac1-236c-4fee-a892-0783c68abbe2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:32:32Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-6e423 namespace: kserve-ci-e2e-test resourceVersion: "27551" uid: 5b767a2b-a0fc-430d-9f27-437a5a87e2e5 kind: Event lastTimestamp: "2026-07-20T10:32:32Z" message: Scaled up replica set splitter-graph-6e423-b7859bb99 from 0 to 1 metadata: creationTimestamp: "2026-07-20T10:32:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:32:32Z" name: splitter-graph-6e423.18c3f8e7e16bf539 namespace: kserve-ci-e2e-test resourceVersion: "27555" uid: 2ab21261-517f-4337-8755-c63cc53f53be reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:32:32Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-6e423 namespace: kserve-ci-e2e-test resourceVersion: "27567" uid: 82b4b8b4-5356-45aa-8015-b510a764215f kind: Event lastTimestamp: "2026-07-20T10:32:32Z" message: InferenceGraph [splitter-graph-6e423] is Ready metadata: creationTimestamp: "2026-07-20T10:32:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:32:32Z" name: splitter-graph-6e423.18c3f8e7ec1f6f77 namespace: kserve-ci-e2e-test resourceVersion: "27569" uid: cf130e2e-d483-4bb1-8e15-76e2d55d016d reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-20T10:32:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-6e423 namespace: kserve-ci-e2e-test resourceVersion: "27552" uid: 2ac3a6ce-9815-4fc9-8ebf-702905db48bf kind: Event lastTimestamp: "2026-07-20T10:33:02Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:32:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:33:02Z" name: splitter-graph-6e423.18c3f8eb6019a8e4 namespace: kserve-ci-e2e-test resourceVersion: "27803" uid: 2214dab6-06bd-484f-9bed-08f07e0d806c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-20T10:32:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-6e423 namespace: kserve-ci-e2e-test resourceVersion: "27552" uid: 2ac3a6ce-9815-4fc9-8ebf-702905db48bf kind: Event lastTimestamp: "2026-07-20T10:33:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:32:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:33:02Z" name: splitter-graph-6e423.18c3f8eb60cd02f0 namespace: kserve-ci-e2e-test resourceVersion: "27804" uid: 187e8bd6-d641-4452-9f38-2921f1703a21 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-20T10:31:11.788444Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-a956e-7c868cfd4b-gl5tm namespace: kserve-ci-e2e-test resourceVersion: "26610" uid: 4705da1d-73a1-4c89-bd08-2bb501de3644 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-a956e-7c868cfd4b-gl5tm to ip-10-0-138-143.ec2.internal metadata: creationTimestamp: "2026-07-20T10:31:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-20T10:31:11Z" name: splitter-graph-a956e-7c868cfd4b-gl5tm.18c3f8d52a31766d namespace: kserve-ci-e2e-test resourceVersion: "26618" uid: cc6ee170-93c1-441e-9c77-76ee95141aad reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5dc669d9-69hkz source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:11Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-a956e-7c868cfd4b-gl5tm namespace: kserve-ci-e2e-test resourceVersion: "26613" uid: 4705da1d-73a1-4c89-bd08-2bb501de3644 kind: Event lastTimestamp: "2026-07-20T10:31:11Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-a956e-serving-cert" not found' metadata: creationTimestamp: "2026-07-20T10:31:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:31:11Z" name: splitter-graph-a956e-7c868cfd4b-gl5tm.18c3f8d535578382 namespace: kserve-ci-e2e-test resourceVersion: "26640" uid: d9b81e6e-d2eb-4a82-897a-c4df5ba4cb5c reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:12Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-a956e-7c868cfd4b-gl5tm namespace: kserve-ci-e2e-test resourceVersion: "26622" uid: 4705da1d-73a1-4c89-bd08-2bb501de3644 kind: Event lastTimestamp: "2026-07-20T10:31:12Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-20T10:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-20T10:31:12Z" name: splitter-graph-a956e-7c868cfd4b-gl5tm.18c3f8d5678bc8ce namespace: kserve-ci-e2e-test resourceVersion: "26659" uid: d9801c72-7c32-451f-a400-a4fc2e454467 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-a956e} kind: Pod name: splitter-graph-a956e-7c868cfd4b-gl5tm namespace: kserve-ci-e2e-test resourceVersion: "26613" uid: 4705da1d-73a1-4c89-bd08-2bb501de3644 kind: Event lastTimestamp: "2026-07-20T10:31:12Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:ee73d7a22df8b79aa8a5e917cb435b1d2b100d4582439c437bdc01a1d2422290" already present on machine metadata: creationTimestamp: "2026-07-20T10:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:31:12Z" name: splitter-graph-a956e-7c868cfd4b-gl5tm.18c3f8d568f0986b namespace: kserve-ci-e2e-test resourceVersion: "26666" uid: e71739db-bad5-4cb8-8ede-3b27d226cfac reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-a956e} kind: Pod name: splitter-graph-a956e-7c868cfd4b-gl5tm namespace: kserve-ci-e2e-test resourceVersion: "26613" uid: 4705da1d-73a1-4c89-bd08-2bb501de3644 kind: Event lastTimestamp: "2026-07-20T10:31:12Z" message: 'Created container: splitter-graph-a956e' metadata: creationTimestamp: "2026-07-20T10:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:31:12Z" name: splitter-graph-a956e-7c868cfd4b-gl5tm.18c3f8d56bf78326 namespace: kserve-ci-e2e-test resourceVersion: "26670" uid: 6dbcafed-2f1c-4d1d-b1e0-375a56128e2c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-a956e} kind: Pod name: splitter-graph-a956e-7c868cfd4b-gl5tm namespace: kserve-ci-e2e-test resourceVersion: "26613" uid: 4705da1d-73a1-4c89-bd08-2bb501de3644 kind: Event lastTimestamp: "2026-07-20T10:31:12Z" message: Started container splitter-graph-a956e metadata: creationTimestamp: "2026-07-20T10:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:31:12Z" name: splitter-graph-a956e-7c868cfd4b-gl5tm.18c3f8d56c6165b7 namespace: kserve-ci-e2e-test resourceVersion: "26671" uid: d3af4f12-306d-4605-8de5-ffd830229637 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-a956e} kind: Pod name: splitter-graph-a956e-7c868cfd4b-gl5tm namespace: kserve-ci-e2e-test resourceVersion: "26613" uid: 4705da1d-73a1-4c89-bd08-2bb501de3644 kind: Event lastTimestamp: "2026-07-20T10:31:21Z" message: Stopping container splitter-graph-a956e metadata: creationTimestamp: "2026-07-20T10:31:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:31:21Z" name: splitter-graph-a956e-7c868cfd4b-gl5tm.18c3f8d78389c9ff namespace: kserve-ci-e2e-test resourceVersion: "26767" uid: 15baaca5-02fd-4dbd-9c0c-94d02c67ba95 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-20T10:31:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-a956e} kind: Pod name: splitter-graph-a956e-7c868cfd4b-gl5tm namespace: kserve-ci-e2e-test resourceVersion: "26613" uid: 4705da1d-73a1-4c89-bd08-2bb501de3644 kind: Event lastTimestamp: "2026-07-20T10:31:49Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-20T10:31:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:31:49Z" name: splitter-graph-a956e-7c868cfd4b-gl5tm.18c3f8d819582400 namespace: kserve-ci-e2e-test resourceVersion: "27133" uid: 96d69a4e-0eae-4f99-a404-1ed5bdce8c66 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-a956e-7c868cfd4b namespace: kserve-ci-e2e-test resourceVersion: "26607" uid: 256c20f2-6a97-494e-8990-e69b1d4fb1de kind: Event lastTimestamp: "2026-07-20T10:31:11Z" message: 'Created pod: splitter-graph-a956e-7c868cfd4b-gl5tm' metadata: creationTimestamp: "2026-07-20T10:31:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:31:11Z" name: splitter-graph-a956e-7c868cfd4b.18c3f8d529a68b2a namespace: kserve-ci-e2e-test resourceVersion: "26611" uid: 1038d727-4242-408c-bbd0-36f572320af9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-a956e namespace: kserve-ci-e2e-test resourceVersion: "26605" uid: c9e10fec-3541-4449-9141-8c0f5c6b26b8 kind: Event lastTimestamp: "2026-07-20T10:31:11Z" message: Scaled up replica set splitter-graph-a956e-7c868cfd4b from 0 to 1 metadata: creationTimestamp: "2026-07-20T10:31:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:31:11Z" name: splitter-graph-a956e.18c3f8d5290c4c5a namespace: kserve-ci-e2e-test resourceVersion: "26608" uid: 0b1d0b16-33d9-4a0c-afce-d010111cb95c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:11Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-a956e namespace: kserve-ci-e2e-test resourceVersion: "26625" uid: df5f1603-fa04-42c9-a53f-b25469bfdcbb kind: Event lastTimestamp: "2026-07-20T10:31:11Z" message: InferenceGraph [splitter-graph-a956e] is Ready metadata: creationTimestamp: "2026-07-20T10:31:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:31:11Z" name: splitter-graph-a956e.18c3f8d52e4ab508 namespace: kserve-ci-e2e-test resourceVersion: "26628" uid: f8ad1022-1b9f-451a-87f7-6d13fa2a5ffe reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:11Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-a956e namespace: kserve-ci-e2e-test resourceVersion: "26596" uid: df5f1603-fa04-42c9-a53f-b25469bfdcbb kind: Event lastTimestamp: "2026-07-20T10:31:11Z" message: 'Failed to update status for InferenceGraph "splitter-graph-a956e": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "splitter-graph-a956e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-20T10:31:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:31:11Z" name: splitter-graph-a956e.18c3f8d5307440dd namespace: kserve-ci-e2e-test resourceVersion: "26635" uid: 9c13454d-8b8c-4497-9ce0-417c19c9a657 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:11Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-a956e namespace: kserve-ci-e2e-test resourceVersion: "26596" uid: df5f1603-fa04-42c9-a53f-b25469bfdcbb kind: Event lastTimestamp: "2026-07-20T10:31:11Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "splitter-graph-a956e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-20T10:31:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:31:11Z" name: splitter-graph-a956e.18c3f8d53074a06f namespace: kserve-ci-e2e-test resourceVersion: "26636" uid: 72ce3495-cb1a-4807-b4c6-a6c9cd315778 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-20T10:19:46.638308Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-242a5-predictor-9b5488df5-wlx2h namespace: kserve-ci-e2e-test resourceVersion: "20325" uid: 2aa656b6-0cc5-43db-99d7-e208cbf41d7a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-242a5-predictor-9b5488df5-wlx2h to ip-10-0-143-155.ec2.internal metadata: creationTimestamp: "2026-07-20T10:19:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-20T10:19:46Z" name: success-200-isvc-242a5-predictor-9b5488df5-wlx2h.18c3f835a40ff964 namespace: kserve-ci-e2e-test resourceVersion: "20342" uid: 14ea77bc-d8d9-4881-9953-222aa24ff746 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5dc669d9-69hkz source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:19:47Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-242a5-predictor-9b5488df5-wlx2h namespace: kserve-ci-e2e-test resourceVersion: "20356" uid: 2aa656b6-0cc5-43db-99d7-e208cbf41d7a kind: Event lastTimestamp: "2026-07-20T10:19:47Z" message: Add eth0 [10.133.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-20T10:19:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-20T10:19:47Z" name: success-200-isvc-242a5-predictor-9b5488df5-wlx2h.18c3f835bdf0363f namespace: kserve-ci-e2e-test resourceVersion: "20399" uid: 6756430d-4a90-44ef-9188-6aae1be57128 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:19:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-242a5-predictor-9b5488df5-wlx2h namespace: kserve-ci-e2e-test resourceVersion: "20330" uid: 2aa656b6-0cc5-43db-99d7-e208cbf41d7a kind: Event lastTimestamp: "2026-07-20T10:19:47Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1771" already present on machine metadata: creationTimestamp: "2026-07-20T10:19:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:19:47Z" name: success-200-isvc-242a5-predictor-9b5488df5-wlx2h.18c3f835c093e709 namespace: kserve-ci-e2e-test resourceVersion: "20401" uid: 9e00c10c-1bce-42f2-a4a3-ef90f4ac32bf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:19:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-242a5-predictor-9b5488df5-wlx2h namespace: kserve-ci-e2e-test resourceVersion: "20330" uid: 2aa656b6-0cc5-43db-99d7-e208cbf41d7a kind: Event lastTimestamp: "2026-07-20T10:19:47Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-20T10:19:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:19:47Z" name: success-200-isvc-242a5-predictor-9b5488df5-wlx2h.18c3f835c41adc4e namespace: kserve-ci-e2e-test resourceVersion: "20403" uid: f3287cc0-8106-4492-a3a4-644056922eea reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:19:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-242a5-predictor-9b5488df5-wlx2h namespace: kserve-ci-e2e-test resourceVersion: "20330" uid: 2aa656b6-0cc5-43db-99d7-e208cbf41d7a kind: Event lastTimestamp: "2026-07-20T10:19:47Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-20T10:19:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:19:47Z" name: success-200-isvc-242a5-predictor-9b5488df5-wlx2h.18c3f835c4971722 namespace: kserve-ci-e2e-test resourceVersion: "20404" uid: e6d39797-b237-46d7-9858-43c4a3fce297 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:19:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-242a5-predictor-9b5488df5-wlx2h namespace: kserve-ci-e2e-test resourceVersion: "20330" uid: 2aa656b6-0cc5-43db-99d7-e208cbf41d7a kind: Event lastTimestamp: "2026-07-20T10:19:47Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-20T10:19:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:19:47Z" name: success-200-isvc-242a5-predictor-9b5488df5-wlx2h.18c3f835c4a0ef92 namespace: kserve-ci-e2e-test resourceVersion: "20406" uid: fe5e9a76-3b86-4073-868d-9897145c74bd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:19:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-242a5-predictor-9b5488df5-wlx2h namespace: kserve-ci-e2e-test resourceVersion: "20330" uid: 2aa656b6-0cc5-43db-99d7-e208cbf41d7a kind: Event lastTimestamp: "2026-07-20T10:19:47Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-20T10:19:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:19:47Z" name: success-200-isvc-242a5-predictor-9b5488df5-wlx2h.18c3f835c7eb7e99 namespace: kserve-ci-e2e-test resourceVersion: "20409" uid: 483c0b93-15f0-4688-8613-fddde917701b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:19:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-242a5-predictor-9b5488df5-wlx2h namespace: kserve-ci-e2e-test resourceVersion: "20330" uid: 2aa656b6-0cc5-43db-99d7-e208cbf41d7a kind: Event lastTimestamp: "2026-07-20T10:19:47Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:19:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:19:47Z" name: success-200-isvc-242a5-predictor-9b5488df5-wlx2h.18c3f835c871126a namespace: kserve-ci-e2e-test resourceVersion: "20410" uid: 1978ecd9-841a-49c0-a648-50a62e8f64c4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-20T10:19:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-242a5-predictor-9b5488df5-wlx2h namespace: kserve-ci-e2e-test resourceVersion: "20330" uid: 2aa656b6-0cc5-43db-99d7-e208cbf41d7a kind: Event lastTimestamp: "2026-07-20T10:20:23Z" message: 'Readiness probe failed: dial tcp 10.133.0.42:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-20T10:19:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:20:23Z" name: success-200-isvc-242a5-predictor-9b5488df5-wlx2h.18c3f835d48fdd6d namespace: kserve-ci-e2e-test resourceVersion: "20710" uid: 1bc9d5e5-0a99-4b4b-87aa-97f46d6d9dc8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-242a5-predictor-9b5488df5-wlx2h namespace: kserve-ci-e2e-test resourceVersion: "20330" uid: 2aa656b6-0cc5-43db-99d7-e208cbf41d7a kind: Event lastTimestamp: "2026-07-20T10:20:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-20T10:20:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:20:56Z" name: success-200-isvc-242a5-predictor-9b5488df5-wlx2h.18c3f845f806917b namespace: kserve-ci-e2e-test resourceVersion: "21234" uid: 1cf805bc-87bd-45df-b58f-47135a1ddf43 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-242a5-predictor-9b5488df5-wlx2h namespace: kserve-ci-e2e-test resourceVersion: "20330" uid: 2aa656b6-0cc5-43db-99d7-e208cbf41d7a kind: Event lastTimestamp: "2026-07-20T10:20:56Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:20:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:20:56Z" name: success-200-isvc-242a5-predictor-9b5488df5-wlx2h.18c3f845f8070b14 namespace: kserve-ci-e2e-test resourceVersion: "21238" uid: 5598d257-672c-4a27-bc22-66d5df13680c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-242a5-predictor-9b5488df5-wlx2h namespace: kserve-ci-e2e-test resourceVersion: "20330" uid: 2aa656b6-0cc5-43db-99d7-e208cbf41d7a kind: Event lastTimestamp: "2026-07-20T10:20:58Z" message: 'Readiness probe failed: Get "https://10.133.0.42:8643/healthz": dial tcp 10.133.0.42:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-20T10:20:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:20:58Z" name: success-200-isvc-242a5-predictor-9b5488df5-wlx2h.18c3f8465d430273 namespace: kserve-ci-e2e-test resourceVersion: "21327" uid: 9c29af15-80e7-43ac-870a-727267b2d6f2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:19:46Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-242a5-predictor-9b5488df5 namespace: kserve-ci-e2e-test resourceVersion: "20315" uid: 03bd3ffb-5b42-4589-86a4-7dcc5f22408c kind: Event lastTimestamp: "2026-07-20T10:19:46Z" message: 'Created pod: success-200-isvc-242a5-predictor-9b5488df5-wlx2h' metadata: creationTimestamp: "2026-07-20T10:19:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:19:46Z" name: success-200-isvc-242a5-predictor-9b5488df5.18c3f835a2ff51c4 namespace: kserve-ci-e2e-test resourceVersion: "20332" uid: c595c470-6d0b-4323-90a2-4559620c51de reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:19:46Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-242a5-predictor namespace: kserve-ci-e2e-test resourceVersion: "20312" uid: 30728520-8ba9-4253-9dce-4a8ce1de1898 kind: Event lastTimestamp: "2026-07-20T10:19:46Z" message: Scaled up replica set success-200-isvc-242a5-predictor-9b5488df5 from 0 to 1 metadata: creationTimestamp: "2026-07-20T10:19:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:19:46Z" name: success-200-isvc-242a5-predictor.18c3f8359dc78c08 namespace: kserve-ci-e2e-test resourceVersion: "20316" uid: a1607f5e-dd42-4354-bc2d-b3c0459ad79e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-20T10:20:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-242a5-predictor namespace: kserve-ci-e2e-test resourceVersion: "20314" uid: ce38acd5-9305-4846-a16f-0c32db958c05 kind: Event lastTimestamp: "2026-07-20T10:20:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:20:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:20:46Z" name: success-200-isvc-242a5-predictor.18c3f8391c085eb2 namespace: kserve-ci-e2e-test resourceVersion: "21069" uid: d0faf911-17f1-4fb1-bb92-dccb82796df9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-20T10:20:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-242a5-predictor namespace: kserve-ci-e2e-test resourceVersion: "20314" uid: ce38acd5-9305-4846-a16f-0c32db958c05 kind: Event lastTimestamp: "2026-07-20T10:20:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:20:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:20:46Z" name: success-200-isvc-242a5-predictor.18c3f8391c7336ab namespace: kserve-ci-e2e-test resourceVersion: "21071" uid: 37125840-b4cf-4ea9-a665-469b857dccbe reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-20T10:19:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-242a5 namespace: kserve-ci-e2e-test resourceVersion: "20304" uid: db471b72-8927-4b6c-adc2-d0c9aeb6bfc0 kind: Event lastTimestamp: "2026-07-20T10:20:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-20T10:19:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:20:33Z" name: success-200-isvc-242a5.18c3f8359dbbc9ef namespace: kserve-ci-e2e-test resourceVersion: "20797" uid: 01557bf7-e6d7-42a8-b01b-fcb5f123b211 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:19:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-242a5 namespace: kserve-ci-e2e-test resourceVersion: "20304" uid: db471b72-8927-4b6c-adc2-d0c9aeb6bfc0 kind: Event lastTimestamp: "2026-07-20T10:19:46Z" message: 'Failed to update status for InferenceService "success-200-isvc-242a5": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-242a5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-20T10:19:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:19:46Z" name: success-200-isvc-242a5.18c3f8359e86794f namespace: kserve-ci-e2e-test resourceVersion: "20319" uid: 389fa5f9-1463-403d-a4db-327322613873 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:19:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-242a5 namespace: kserve-ci-e2e-test resourceVersion: "20304" uid: db471b72-8927-4b6c-adc2-d0c9aeb6bfc0 kind: Event lastTimestamp: "2026-07-20T10:19:46Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-242a5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-20T10:19:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:19:46Z" name: success-200-isvc-242a5.18c3f8359e86aa94 namespace: kserve-ci-e2e-test resourceVersion: "20321" uid: df6050a6-3974-4f4b-b209-35f2e8080a03 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-242a5 namespace: kserve-ci-e2e-test resourceVersion: "20791" uid: db471b72-8927-4b6c-adc2-d0c9aeb6bfc0 kind: Event lastTimestamp: "2026-07-20T10:20:33Z" message: InferenceService [success-200-isvc-242a5] is Ready metadata: creationTimestamp: "2026-07-20T10:20:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:20:33Z" name: success-200-isvc-242a5.18c3f8408fe03345 namespace: kserve-ci-e2e-test resourceVersion: "20792" uid: 5a614631-872c-4146-b75d-db9ce61f1cf3 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-20T10:20:36.226461Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-24b48-predictor-7d8fb7d579-wz4wf namespace: kserve-ci-e2e-test resourceVersion: "20884" uid: 6326a41c-0694-4a0d-992d-69854cc8802b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-24b48-predictor-7d8fb7d579-wz4wf to ip-10-0-143-155.ec2.internal metadata: creationTimestamp: "2026-07-20T10:20:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-20T10:20:36Z" name: success-200-isvc-24b48-predictor-7d8fb7d579-wz4wf.18c3f8412fbf2db1 namespace: kserve-ci-e2e-test resourceVersion: "20890" uid: 36cbc02a-eef4-4538-b816-f47ffef4ee8c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5dc669d9-69hkz source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:36Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-24b48-predictor-7d8fb7d579-wz4wf namespace: kserve-ci-e2e-test resourceVersion: "20887" uid: 6326a41c-0694-4a0d-992d-69854cc8802b kind: Event lastTimestamp: "2026-07-20T10:20:36Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-24b48-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-20T10:20:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:20:36Z" name: success-200-isvc-24b48-predictor-7d8fb7d579-wz4wf.18c3f8413b459f26 namespace: kserve-ci-e2e-test resourceVersion: "20928" uid: c2930cf3-3495-46c7-93c4-14978e052461 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:37Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-24b48-predictor-7d8fb7d579-wz4wf namespace: kserve-ci-e2e-test resourceVersion: "20895" uid: 6326a41c-0694-4a0d-992d-69854cc8802b kind: Event lastTimestamp: "2026-07-20T10:20:37Z" message: Add eth0 [10.133.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-20T10:20:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-20T10:20:37Z" name: success-200-isvc-24b48-predictor-7d8fb7d579-wz4wf.18c3f8416d534fac namespace: kserve-ci-e2e-test resourceVersion: "20940" uid: e4cb1213-7f86-4434-8150-47eac015a263 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-24b48-predictor-7d8fb7d579-wz4wf namespace: kserve-ci-e2e-test resourceVersion: "20887" uid: 6326a41c-0694-4a0d-992d-69854cc8802b kind: Event lastTimestamp: "2026-07-20T10:20:37Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1771" already present on machine metadata: creationTimestamp: "2026-07-20T10:20:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:20:37Z" name: success-200-isvc-24b48-predictor-7d8fb7d579-wz4wf.18c3f8416f05a6b3 namespace: kserve-ci-e2e-test resourceVersion: "20943" uid: 586b78a0-f579-49be-90c3-45e44eb00c45 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-24b48-predictor-7d8fb7d579-wz4wf namespace: kserve-ci-e2e-test resourceVersion: "20887" uid: 6326a41c-0694-4a0d-992d-69854cc8802b kind: Event lastTimestamp: "2026-07-20T10:20:37Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-20T10:20:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:20:37Z" name: success-200-isvc-24b48-predictor-7d8fb7d579-wz4wf.18c3f841733571eb namespace: kserve-ci-e2e-test resourceVersion: "20944" uid: b982ae1c-8430-45f0-8708-025e7d473220 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-24b48-predictor-7d8fb7d579-wz4wf namespace: kserve-ci-e2e-test resourceVersion: "20887" uid: 6326a41c-0694-4a0d-992d-69854cc8802b kind: Event lastTimestamp: "2026-07-20T10:20:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-20T10:20:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:20:37Z" name: success-200-isvc-24b48-predictor-7d8fb7d579-wz4wf.18c3f84173daa406 namespace: kserve-ci-e2e-test resourceVersion: "20945" uid: eab2d480-cad6-4002-af47-5dfb96840341 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-24b48-predictor-7d8fb7d579-wz4wf namespace: kserve-ci-e2e-test resourceVersion: "20887" uid: 6326a41c-0694-4a0d-992d-69854cc8802b kind: Event lastTimestamp: "2026-07-20T10:20:37Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-20T10:20:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:20:37Z" name: success-200-isvc-24b48-predictor-7d8fb7d579-wz4wf.18c3f84173e62aeb namespace: kserve-ci-e2e-test resourceVersion: "20946" uid: f390988d-ab7e-4e92-9d72-b8e24205c895 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-24b48-predictor-7d8fb7d579-wz4wf namespace: kserve-ci-e2e-test resourceVersion: "20887" uid: 6326a41c-0694-4a0d-992d-69854cc8802b kind: Event lastTimestamp: "2026-07-20T10:20:37Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-20T10:20:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:20:37Z" name: success-200-isvc-24b48-predictor-7d8fb7d579-wz4wf.18c3f84176b9fef3 namespace: kserve-ci-e2e-test resourceVersion: "20950" uid: bd0d054f-ed12-4135-ab4d-cc4648b41d18 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-24b48-predictor-7d8fb7d579-wz4wf namespace: kserve-ci-e2e-test resourceVersion: "20887" uid: 6326a41c-0694-4a0d-992d-69854cc8802b kind: Event lastTimestamp: "2026-07-20T10:20:37Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:20:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:20:37Z" name: success-200-isvc-24b48-predictor-7d8fb7d579-wz4wf.18c3f84177445e1d namespace: kserve-ci-e2e-test resourceVersion: "20951" uid: c9612a94-858d-498d-a2a8-6720474dece8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-20T10:20:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-24b48-predictor-7d8fb7d579-wz4wf namespace: kserve-ci-e2e-test resourceVersion: "20887" uid: 6326a41c-0694-4a0d-992d-69854cc8802b kind: Event lastTimestamp: "2026-07-20T10:21:23Z" message: 'Readiness probe failed: dial tcp 10.133.0.44:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-20T10:20:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:21:23Z" name: success-200-isvc-24b48-predictor-7d8fb7d579-wz4wf.18c3f84188eb799d namespace: kserve-ci-e2e-test resourceVersion: "21541" uid: 4bc36f6d-1f49-4b81-8984-a789f88c681c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-24b48-predictor-7d8fb7d579-wz4wf namespace: kserve-ci-e2e-test resourceVersion: "20887" uid: 6326a41c-0694-4a0d-992d-69854cc8802b kind: Event lastTimestamp: "2026-07-20T10:21:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-20T10:21:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:21:56Z" name: success-200-isvc-24b48-predictor-7d8fb7d579-wz4wf.18c3f853dcb6d091 namespace: kserve-ci-e2e-test resourceVersion: "21900" uid: b64147a7-6652-4fea-bfb8-d16f0fc0ca74 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-24b48-predictor-7d8fb7d579-wz4wf namespace: kserve-ci-e2e-test resourceVersion: "20887" uid: 6326a41c-0694-4a0d-992d-69854cc8802b kind: Event lastTimestamp: "2026-07-20T10:21:56Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:21:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:21:56Z" name: success-200-isvc-24b48-predictor-7d8fb7d579-wz4wf.18c3f853dcb7e4a8 namespace: kserve-ci-e2e-test resourceVersion: "21902" uid: f6d5600c-8136-43ee-b154-2e8091859f5e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-24b48-predictor-7d8fb7d579-wz4wf namespace: kserve-ci-e2e-test resourceVersion: "20887" uid: 6326a41c-0694-4a0d-992d-69854cc8802b kind: Event lastTimestamp: "2026-07-20T10:21:58Z" message: 'Readiness probe failed: Get "https://10.133.0.44:8643/healthz": dial tcp 10.133.0.44:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-20T10:21:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:21:58Z" name: success-200-isvc-24b48-predictor-7d8fb7d579-wz4wf.18c3f854651c66c4 namespace: kserve-ci-e2e-test resourceVersion: "22062" uid: 83abbcf2-7f78-464e-8695-45ca5ae53ff0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-24b48-predictor-7d8fb7d579 namespace: kserve-ci-e2e-test resourceVersion: "20875" uid: 864f632a-1ff7-4d81-b5c9-509b660c69ab kind: Event lastTimestamp: "2026-07-20T10:20:36Z" message: 'Created pod: success-200-isvc-24b48-predictor-7d8fb7d579-wz4wf' metadata: creationTimestamp: "2026-07-20T10:20:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:20:36Z" name: success-200-isvc-24b48-predictor-7d8fb7d579.18c3f8412f34a37b namespace: kserve-ci-e2e-test resourceVersion: "20886" uid: d1b42872-8b6b-44bd-a926-2d0f17167eaf reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-24b48-predictor namespace: kserve-ci-e2e-test resourceVersion: "20872" uid: b2b79fc5-9a4b-4f27-810f-d3d14618b927 kind: Event lastTimestamp: "2026-07-20T10:20:36Z" message: Scaled up replica set success-200-isvc-24b48-predictor-7d8fb7d579 from 0 to 1 metadata: creationTimestamp: "2026-07-20T10:20:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:20:36Z" name: success-200-isvc-24b48-predictor.18c3f8412d8649dd namespace: kserve-ci-e2e-test resourceVersion: "20877" uid: f150603c-0743-4760-bd12-4ed66ea9fa4c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-20T10:20:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-24b48-predictor namespace: kserve-ci-e2e-test resourceVersion: "20873" uid: 73161d1d-a3a6-4f26-82db-22878962f4d3 kind: Event lastTimestamp: "2026-07-20T10:21:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:20:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:21:06Z" name: success-200-isvc-24b48-predictor.18c3f844abe22d88 namespace: kserve-ci-e2e-test resourceVersion: "21393" uid: 1805f7cd-b6ec-4b03-b3a9-3c434b8e2f9c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-20T10:20:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-24b48-predictor namespace: kserve-ci-e2e-test resourceVersion: "20873" uid: 73161d1d-a3a6-4f26-82db-22878962f4d3 kind: Event lastTimestamp: "2026-07-20T10:21:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:20:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:21:06Z" name: success-200-isvc-24b48-predictor.18c3f844ac51130f namespace: kserve-ci-e2e-test resourceVersion: "21394" uid: 5070e09d-616e-47e7-87d7-bff6f2776ca8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-20T10:21:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-24b48-predictor namespace: kserve-ci-e2e-test resourceVersion: "21126" uid: 73161d1d-a3a6-4f26-82db-22878962f4d3 kind: Event lastTimestamp: "2026-07-20T10:21:51Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:21:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:21:51Z" name: success-200-isvc-24b48-predictor.18c3f84baa4ee91d namespace: kserve-ci-e2e-test resourceVersion: "21824" uid: 7e6939c2-945f-46b7-998b-0e0fbff5839a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-20T10:21:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-24b48-predictor namespace: kserve-ci-e2e-test resourceVersion: "21126" uid: 73161d1d-a3a6-4f26-82db-22878962f4d3 kind: Event lastTimestamp: "2026-07-20T10:21:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:21:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:21:51Z" name: success-200-isvc-24b48-predictor.18c3f84baab3c921 namespace: kserve-ci-e2e-test resourceVersion: "21825" uid: 410fe03a-4700-42aa-bfee-f61e4b889668 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-20T10:20:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-24b48 namespace: kserve-ci-e2e-test resourceVersion: "20848" uid: bc60eeb2-89d0-4730-b892-c5cd807003a8 kind: Event lastTimestamp: "2026-07-20T10:21:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-20T10:20:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:21:33Z" name: success-200-isvc-24b48.18c3f8412d8ae5be namespace: kserve-ci-e2e-test resourceVersion: "21639" uid: 3b671135-b83c-402b-aa35-6ffb1839f72c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-24b48 namespace: kserve-ci-e2e-test resourceVersion: "20848" uid: bc60eeb2-89d0-4730-b892-c5cd807003a8 kind: Event lastTimestamp: "2026-07-20T10:20:36Z" message: 'Failed to update status for InferenceService "success-200-isvc-24b48": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-24b48": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-20T10:20:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:20:36Z" name: success-200-isvc-24b48.18c3f8412e2a109c namespace: kserve-ci-e2e-test resourceVersion: "20881" uid: 71be25b0-5796-430b-a06e-f9c9990b2415 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-24b48 namespace: kserve-ci-e2e-test resourceVersion: "20848" uid: bc60eeb2-89d0-4730-b892-c5cd807003a8 kind: Event lastTimestamp: "2026-07-20T10:20:36Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-24b48": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-20T10:20:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:20:36Z" name: success-200-isvc-24b48.18c3f8412e2a5863 namespace: kserve-ci-e2e-test resourceVersion: "20882" uid: e0f55440-c0f8-4059-89c2-6c4044442a4e reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-24b48 namespace: kserve-ci-e2e-test resourceVersion: "21636" uid: bc60eeb2-89d0-4730-b892-c5cd807003a8 kind: Event lastTimestamp: "2026-07-20T10:21:33Z" message: InferenceService [success-200-isvc-24b48] is Ready metadata: creationTimestamp: "2026-07-20T10:21:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:21:33Z" name: success-200-isvc-24b48.18c3f84e9c72c050 namespace: kserve-ci-e2e-test resourceVersion: "21637" uid: 79ae4e8b-9a01-42b8-abe1-4c68e78420f3 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-20T10:20:56.809168Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-53302-predictor-687d4d7db9-w9qtn namespace: kserve-ci-e2e-test resourceVersion: "21242" uid: 22d1388b-807e-4cbe-836f-b594315998e8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-53302-predictor-687d4d7db9-w9qtn to ip-10-0-143-155.ec2.internal metadata: creationTimestamp: "2026-07-20T10:20:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-20T10:20:56Z" name: success-200-isvc-53302-predictor-687d4d7db9-w9qtn.18c3f845fa925916 namespace: kserve-ci-e2e-test resourceVersion: "21247" uid: e3e56a00-b378-45d2-8580-19bf5a7dd545 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5dc669d9-69hkz source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:57Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-53302-predictor-687d4d7db9-w9qtn namespace: kserve-ci-e2e-test resourceVersion: "21257" uid: 22d1388b-807e-4cbe-836f-b594315998e8 kind: Event lastTimestamp: "2026-07-20T10:20:57Z" message: Add eth0 [10.133.0.46/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-20T10:20:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-20T10:20:57Z" name: success-200-isvc-53302-predictor-687d4d7db9-w9qtn.18c3f84614748452 namespace: kserve-ci-e2e-test resourceVersion: "21290" uid: 739d0b01-2fa8-4424-996b-1f4c945e148c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-53302-predictor-687d4d7db9-w9qtn namespace: kserve-ci-e2e-test resourceVersion: "21245" uid: 22d1388b-807e-4cbe-836f-b594315998e8 kind: Event lastTimestamp: "2026-07-20T10:20:57Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1771" already present on machine metadata: creationTimestamp: "2026-07-20T10:20:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:20:57Z" name: success-200-isvc-53302-predictor-687d4d7db9-w9qtn.18c3f846160a2ef2 namespace: kserve-ci-e2e-test resourceVersion: "21292" uid: c197843b-a40f-4c4c-a58b-08092769c962 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-53302-predictor-687d4d7db9-w9qtn namespace: kserve-ci-e2e-test resourceVersion: "21245" uid: 22d1388b-807e-4cbe-836f-b594315998e8 kind: Event lastTimestamp: "2026-07-20T10:20:57Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-20T10:20:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:20:57Z" name: success-200-isvc-53302-predictor-687d4d7db9-w9qtn.18c3f84619a2242e namespace: kserve-ci-e2e-test resourceVersion: "21293" uid: 31931705-061e-4bf7-8bb3-e2859f44062e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-53302-predictor-687d4d7db9-w9qtn namespace: kserve-ci-e2e-test resourceVersion: "21245" uid: 22d1388b-807e-4cbe-836f-b594315998e8 kind: Event lastTimestamp: "2026-07-20T10:20:57Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-20T10:20:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:20:57Z" name: success-200-isvc-53302-predictor-687d4d7db9-w9qtn.18c3f8461a548c59 namespace: kserve-ci-e2e-test resourceVersion: "21294" uid: 691a29f3-7cd7-4475-925d-a1dcdb83ef79 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-53302-predictor-687d4d7db9-w9qtn namespace: kserve-ci-e2e-test resourceVersion: "21245" uid: 22d1388b-807e-4cbe-836f-b594315998e8 kind: Event lastTimestamp: "2026-07-20T10:20:57Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-20T10:20:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:20:57Z" name: success-200-isvc-53302-predictor-687d4d7db9-w9qtn.18c3f8461a61aeef namespace: kserve-ci-e2e-test resourceVersion: "21295" uid: dc3e2041-7c52-4f24-8646-d3357a3501f4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-53302-predictor-687d4d7db9-w9qtn namespace: kserve-ci-e2e-test resourceVersion: "21245" uid: 22d1388b-807e-4cbe-836f-b594315998e8 kind: Event lastTimestamp: "2026-07-20T10:20:57Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-20T10:20:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:20:57Z" name: success-200-isvc-53302-predictor-687d4d7db9-w9qtn.18c3f8461db70fbd namespace: kserve-ci-e2e-test resourceVersion: "21299" uid: bfdde90d-af95-440f-a47b-6db3f2f87839 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-53302-predictor-687d4d7db9-w9qtn namespace: kserve-ci-e2e-test resourceVersion: "21245" uid: 22d1388b-807e-4cbe-836f-b594315998e8 kind: Event lastTimestamp: "2026-07-20T10:20:57Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:20:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:20:57Z" name: success-200-isvc-53302-predictor-687d4d7db9-w9qtn.18c3f8461e4160fb namespace: kserve-ci-e2e-test resourceVersion: "21300" uid: 458ece58-1c8c-4343-a4d9-18264b1d1242 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-20T10:20:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-53302-predictor-687d4d7db9-w9qtn namespace: kserve-ci-e2e-test resourceVersion: "21245" uid: 22d1388b-807e-4cbe-836f-b594315998e8 kind: Event lastTimestamp: "2026-07-20T10:30:13Z" message: 'Readiness probe failed: dial tcp 10.133.0.46:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-20T10:20:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:30:13Z" name: success-200-isvc-53302-predictor-687d4d7db9-w9qtn.18c3f846375a83a4 namespace: kserve-ci-e2e-test resourceVersion: "26016" uid: bab50487-ab21-4ddf-a6ea-b62329cb8b79 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:30:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-53302-predictor-687d4d7db9-w9qtn namespace: kserve-ci-e2e-test resourceVersion: "21245" uid: 22d1388b-807e-4cbe-836f-b594315998e8 kind: Event lastTimestamp: "2026-07-20T10:30:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-20T10:30:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:30:11Z" name: success-200-isvc-53302-predictor-687d4d7db9-w9qtn.18c3f8c73428a18f namespace: kserve-ci-e2e-test resourceVersion: "25935" uid: 40912f76-23d5-4789-9067-4a988154a1aa reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:30:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-53302-predictor-687d4d7db9-w9qtn namespace: kserve-ci-e2e-test resourceVersion: "21245" uid: 22d1388b-807e-4cbe-836f-b594315998e8 kind: Event lastTimestamp: "2026-07-20T10:30:11Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:30:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:30:11Z" name: success-200-isvc-53302-predictor-687d4d7db9-w9qtn.18c3f8c73429a194 namespace: kserve-ci-e2e-test resourceVersion: "25936" uid: d047792e-b627-4f3b-a45a-f5a736363cf3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:30:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-53302-predictor-687d4d7db9-w9qtn namespace: kserve-ci-e2e-test resourceVersion: "21245" uid: 22d1388b-807e-4cbe-836f-b594315998e8 kind: Event lastTimestamp: "2026-07-20T10:30:13Z" message: 'Readiness probe failed: Get "https://10.133.0.46:8643/healthz": dial tcp 10.133.0.46:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-20T10:30:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:30:13Z" name: success-200-isvc-53302-predictor-687d4d7db9-w9qtn.18c3f8c7abe9582b namespace: kserve-ci-e2e-test resourceVersion: "26015" uid: 76d8eeaa-2f46-40c0-a0c7-6cc4f591447f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-53302-predictor-687d4d7db9 namespace: kserve-ci-e2e-test resourceVersion: "21232" uid: c5481bef-ba68-4071-a05c-4ae7c83a4fe3 kind: Event lastTimestamp: "2026-07-20T10:20:56Z" message: 'Created pod: success-200-isvc-53302-predictor-687d4d7db9-w9qtn' metadata: creationTimestamp: "2026-07-20T10:20:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:20:56Z" name: success-200-isvc-53302-predictor-687d4d7db9.18c3f845f9fbae20 namespace: kserve-ci-e2e-test resourceVersion: "21244" uid: 4271e9df-84d2-4f32-9173-921e1278cb41 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-53302-predictor namespace: kserve-ci-e2e-test resourceVersion: "21227" uid: f9963a42-f980-4af3-91cd-3d196e1b2288 kind: Event lastTimestamp: "2026-07-20T10:20:56Z" message: Scaled up replica set success-200-isvc-53302-predictor-687d4d7db9 from 0 to 1 metadata: creationTimestamp: "2026-07-20T10:20:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:20:56Z" name: success-200-isvc-53302-predictor.18c3f845f80ef0a2 namespace: kserve-ci-e2e-test resourceVersion: "21235" uid: 0a7642ef-eee8-4d67-8e57-126592868840 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-20T10:21:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-53302-predictor namespace: kserve-ci-e2e-test resourceVersion: "21229" uid: cbea983a-54b5-433f-9499-ae258f83b1bc kind: Event lastTimestamp: "2026-07-20T10:21:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:21:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:21:26Z" name: success-200-isvc-53302-predictor.18c3f8497656482d namespace: kserve-ci-e2e-test resourceVersion: "21565" uid: 31244531-7776-4bbe-8a31-fe06f9212a08 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-20T10:21:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-53302-predictor namespace: kserve-ci-e2e-test resourceVersion: "21229" uid: cbea983a-54b5-433f-9499-ae258f83b1bc kind: Event lastTimestamp: "2026-07-20T10:21:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:21:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:21:26Z" name: success-200-isvc-53302-predictor.18c3f84976cb6c01 namespace: kserve-ci-e2e-test resourceVersion: "21566" uid: d9e40549-5d6c-4d44-aefe-f8696da42ff5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-20T10:21:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-53302-predictor namespace: kserve-ci-e2e-test resourceVersion: "21444" uid: cbea983a-54b5-433f-9499-ae258f83b1bc kind: Event lastTimestamp: "2026-07-20T10:22:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:21:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:22:11Z" name: success-200-isvc-53302-predictor.18c3f8507433047f namespace: kserve-ci-e2e-test resourceVersion: "22188" uid: d43a6bcc-bd96-406e-822b-64b3d83854f3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-20T10:21:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-53302-predictor namespace: kserve-ci-e2e-test resourceVersion: "21444" uid: cbea983a-54b5-433f-9499-ae258f83b1bc kind: Event lastTimestamp: "2026-07-20T10:22:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:21:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:22:11Z" name: success-200-isvc-53302-predictor.18c3f850747d8f66 namespace: kserve-ci-e2e-test resourceVersion: "22189" uid: 5ae7fbab-78c3-4e06-b3c0-b84d7838f0fc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-20T10:20:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-53302 namespace: kserve-ci-e2e-test resourceVersion: "21206" uid: fe4d2e80-4e1b-4ed2-bdd2-53a02c5eb130 kind: Event lastTimestamp: "2026-07-20T10:21:44Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-20T10:20:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:21:44Z" name: success-200-isvc-53302.18c3f845f7d2b8fe namespace: kserve-ci-e2e-test resourceVersion: "21741" uid: 314fa372-d4be-4382-b875-42e85140ef90 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-53302 namespace: kserve-ci-e2e-test resourceVersion: "21206" uid: fe4d2e80-4e1b-4ed2-bdd2-53a02c5eb130 kind: Event lastTimestamp: "2026-07-20T10:20:56Z" message: 'Failed to update status for InferenceService "success-200-isvc-53302": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-53302": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-20T10:20:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:20:56Z" name: success-200-isvc-53302.18c3f845f83f3d0f namespace: kserve-ci-e2e-test resourceVersion: "21237" uid: 650240ea-78dd-4d0b-8ba8-c50d6f92232e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-53302 namespace: kserve-ci-e2e-test resourceVersion: "21206" uid: fe4d2e80-4e1b-4ed2-bdd2-53a02c5eb130 kind: Event lastTimestamp: "2026-07-20T10:20:56Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-53302": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-20T10:20:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:20:56Z" name: success-200-isvc-53302.18c3f845f83f9a71 namespace: kserve-ci-e2e-test resourceVersion: "21240" uid: f1c6bd44-b067-4653-b35f-f51dd7920e21 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-53302 namespace: kserve-ci-e2e-test resourceVersion: "21734" uid: fe4d2e80-4e1b-4ed2-bdd2-53a02c5eb130 kind: Event lastTimestamp: "2026-07-20T10:21:44Z" message: InferenceService [success-200-isvc-53302] is Ready metadata: creationTimestamp: "2026-07-20T10:21:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:21:44Z" name: success-200-isvc-53302.18c3f850f86a758b namespace: kserve-ci-e2e-test resourceVersion: "21736" uid: 4752d326-ac99-4e9b-a1c8-6984360a7f5c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-20T10:21:56.531766Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-64e78-predictor-676cf96d7f-fsxql namespace: kserve-ci-e2e-test resourceVersion: "21934" uid: 50e96327-c06e-4abc-b3b7-4c3992a3a34f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-64e78-predictor-676cf96d7f-fsxql to ip-10-0-143-155.ec2.internal metadata: creationTimestamp: "2026-07-20T10:21:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-20T10:21:56Z" name: success-200-isvc-64e78-predictor-676cf96d7f-fsxql.18c3f853e250daf1 namespace: kserve-ci-e2e-test resourceVersion: "21939" uid: 4bc0bf26-c363-4bbb-aac7-9002b6107683 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5dc669d9-69hkz source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:56Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-64e78-predictor-676cf96d7f-fsxql namespace: kserve-ci-e2e-test resourceVersion: "21935" uid: 50e96327-c06e-4abc-b3b7-4c3992a3a34f kind: Event lastTimestamp: "2026-07-20T10:21:56Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-64e78-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-20T10:21:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:21:56Z" name: success-200-isvc-64e78-predictor-676cf96d7f-fsxql.18c3f853f24795ac namespace: kserve-ci-e2e-test resourceVersion: "21980" uid: 39626c5e-393d-4616-8c27-ffb29b78d49c reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:57Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-64e78-predictor-676cf96d7f-fsxql namespace: kserve-ci-e2e-test resourceVersion: "21945" uid: 50e96327-c06e-4abc-b3b7-4c3992a3a34f kind: Event lastTimestamp: "2026-07-20T10:21:57Z" message: Add eth0 [10.133.0.48/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-20T10:21:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-20T10:21:57Z" name: success-200-isvc-64e78-predictor-676cf96d7f-fsxql.18c3f8541fd50218 namespace: kserve-ci-e2e-test resourceVersion: "22030" uid: f7b4434c-9a14-4a3a-9b8e-8068cd8ae8ea reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-64e78-predictor-676cf96d7f-fsxql namespace: kserve-ci-e2e-test resourceVersion: "21935" uid: 50e96327-c06e-4abc-b3b7-4c3992a3a34f kind: Event lastTimestamp: "2026-07-20T10:21:57Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1771" already present on machine metadata: creationTimestamp: "2026-07-20T10:21:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:21:57Z" name: success-200-isvc-64e78-predictor-676cf96d7f-fsxql.18c3f854213c1b3a namespace: kserve-ci-e2e-test resourceVersion: "22032" uid: cf35dc0c-e3d2-432d-858d-771f912f6749 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-64e78-predictor-676cf96d7f-fsxql namespace: kserve-ci-e2e-test resourceVersion: "21935" uid: 50e96327-c06e-4abc-b3b7-4c3992a3a34f kind: Event lastTimestamp: "2026-07-20T10:21:57Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-20T10:21:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:21:57Z" name: success-200-isvc-64e78-predictor-676cf96d7f-fsxql.18c3f85424748494 namespace: kserve-ci-e2e-test resourceVersion: "22034" uid: 2c5506d4-50d9-4524-b1d3-a29dc40cd5aa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-64e78-predictor-676cf96d7f-fsxql namespace: kserve-ci-e2e-test resourceVersion: "21935" uid: 50e96327-c06e-4abc-b3b7-4c3992a3a34f kind: Event lastTimestamp: "2026-07-20T10:21:57Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-20T10:21:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:21:57Z" name: success-200-isvc-64e78-predictor-676cf96d7f-fsxql.18c3f85424dec5cd namespace: kserve-ci-e2e-test resourceVersion: "22035" uid: 133a8c9e-6218-494f-8afa-27ea17b672d0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-64e78-predictor-676cf96d7f-fsxql namespace: kserve-ci-e2e-test resourceVersion: "21935" uid: 50e96327-c06e-4abc-b3b7-4c3992a3a34f kind: Event lastTimestamp: "2026-07-20T10:21:57Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-20T10:21:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:21:57Z" name: success-200-isvc-64e78-predictor-676cf96d7f-fsxql.18c3f85424e88d21 namespace: kserve-ci-e2e-test resourceVersion: "22036" uid: e4317322-c034-4769-98e1-1d3573876c96 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-64e78-predictor-676cf96d7f-fsxql namespace: kserve-ci-e2e-test resourceVersion: "21935" uid: 50e96327-c06e-4abc-b3b7-4c3992a3a34f kind: Event lastTimestamp: "2026-07-20T10:21:57Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-20T10:21:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:21:57Z" name: success-200-isvc-64e78-predictor-676cf96d7f-fsxql.18c3f85427d01772 namespace: kserve-ci-e2e-test resourceVersion: "22037" uid: 7bd07670-0c4b-4a38-9535-a69318c84b0e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-64e78-predictor-676cf96d7f-fsxql namespace: kserve-ci-e2e-test resourceVersion: "21935" uid: 50e96327-c06e-4abc-b3b7-4c3992a3a34f kind: Event lastTimestamp: "2026-07-20T10:21:57Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:21:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:21:57Z" name: success-200-isvc-64e78-predictor-676cf96d7f-fsxql.18c3f8542852d8ec namespace: kserve-ci-e2e-test resourceVersion: "22038" uid: cde1dfed-d383-4fbb-8500-485e1c008270 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-20T10:21:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-64e78-predictor-676cf96d7f-fsxql namespace: kserve-ci-e2e-test resourceVersion: "21935" uid: 50e96327-c06e-4abc-b3b7-4c3992a3a34f kind: Event lastTimestamp: "2026-07-20T10:22:35Z" message: 'Readiness probe failed: dial tcp 10.133.0.48:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-20T10:21:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:22:35Z" name: success-200-isvc-64e78-predictor-676cf96d7f-fsxql.18c3f8547e29f208 namespace: kserve-ci-e2e-test resourceVersion: "22372" uid: 100c2197-1a07-440f-8d11-a8e114c45944 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-64e78-predictor-676cf96d7f-fsxql namespace: kserve-ci-e2e-test resourceVersion: "21935" uid: 50e96327-c06e-4abc-b3b7-4c3992a3a34f kind: Event lastTimestamp: "2026-07-20T10:31:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-20T10:31:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:31:11Z" name: success-200-isvc-64e78-predictor-676cf96d7f-fsxql.18c3f8d50b4d6fe1 namespace: kserve-ci-e2e-test resourceVersion: "26534" uid: 321bf494-4db9-451e-94ad-327cc32fa53c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-64e78-predictor-676cf96d7f-fsxql namespace: kserve-ci-e2e-test resourceVersion: "21935" uid: 50e96327-c06e-4abc-b3b7-4c3992a3a34f kind: Event lastTimestamp: "2026-07-20T10:31:11Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:31:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:31:11Z" name: success-200-isvc-64e78-predictor-676cf96d7f-fsxql.18c3f8d50b4e9660 namespace: kserve-ci-e2e-test resourceVersion: "26537" uid: e405625c-ce6f-4eeb-8217-b60bad3d6f67 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-64e78-predictor-676cf96d7f namespace: kserve-ci-e2e-test resourceVersion: "21925" uid: 9e228614-6e74-4e9b-833c-c3e207167eaa kind: Event lastTimestamp: "2026-07-20T10:21:56Z" message: 'Created pod: success-200-isvc-64e78-predictor-676cf96d7f-fsxql' metadata: creationTimestamp: "2026-07-20T10:21:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:21:56Z" name: success-200-isvc-64e78-predictor-676cf96d7f.18c3f853e1de5d21 namespace: kserve-ci-e2e-test resourceVersion: "21936" uid: aefa13cc-df3a-4d83-8e84-6795efb52a9f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-64e78-predictor namespace: kserve-ci-e2e-test resourceVersion: "21921" uid: 582d76dd-e228-4e8d-8329-31e5fe26b902 kind: Event lastTimestamp: "2026-07-20T10:21:56Z" message: Scaled up replica set success-200-isvc-64e78-predictor-676cf96d7f from 0 to 1 metadata: creationTimestamp: "2026-07-20T10:21:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:21:56Z" name: success-200-isvc-64e78-predictor.18c3f853dfd25cc4 namespace: kserve-ci-e2e-test resourceVersion: "21928" uid: b460b81f-8790-4b31-b2ae-0e4eab55622d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-20T10:22:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-64e78-predictor namespace: kserve-ci-e2e-test resourceVersion: "21923" uid: be80550c-bc8e-460e-a8d0-47c9a2fec4dd kind: Event lastTimestamp: "2026-07-20T10:22:41Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:22:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:22:41Z" name: success-200-isvc-64e78-predictor.18c3f8575ece7f55 namespace: kserve-ci-e2e-test resourceVersion: "22412" uid: ee47928d-eb75-47a2-9743-28c096f4a8b2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-20T10:22:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-64e78-predictor namespace: kserve-ci-e2e-test resourceVersion: "21923" uid: be80550c-bc8e-460e-a8d0-47c9a2fec4dd kind: Event lastTimestamp: "2026-07-20T10:22:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:22:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:22:41Z" name: success-200-isvc-64e78-predictor.18c3f8575f3a605a namespace: kserve-ci-e2e-test resourceVersion: "22413" uid: 0ac0aecb-9592-46ad-8097-c99244c2bb39 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-20T10:22:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-64e78-predictor namespace: kserve-ci-e2e-test resourceVersion: "22180" uid: be80550c-bc8e-460e-a8d0-47c9a2fec4dd kind: Event lastTimestamp: "2026-07-20T10:23:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:22:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:23:26Z" name: success-200-isvc-64e78-predictor.18c3f85ade1a1df2 namespace: kserve-ci-e2e-test resourceVersion: "22802" uid: 82a07f05-bd25-43d7-9c24-7cd6820082ab reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-20T10:22:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-64e78-predictor namespace: kserve-ci-e2e-test resourceVersion: "22180" uid: be80550c-bc8e-460e-a8d0-47c9a2fec4dd kind: Event lastTimestamp: "2026-07-20T10:23:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:22:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:23:26Z" name: success-200-isvc-64e78-predictor.18c3f85ade8b01b7 namespace: kserve-ci-e2e-test resourceVersion: "22803" uid: 6a096aa8-8de1-451d-8f8c-ef1bb5d44100 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-20T10:21:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-64e78 namespace: kserve-ci-e2e-test resourceVersion: "21898" uid: 1134385d-5e64-47b7-b7f6-c78d2df09c77 kind: Event lastTimestamp: "2026-07-20T10:22:45Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-20T10:21:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:22:45Z" name: success-200-isvc-64e78.18c3f853dfd03ba3 namespace: kserve-ci-e2e-test resourceVersion: "22461" uid: de2127f6-d14c-462a-8f3f-e52f9d8d2691 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-64e78 namespace: kserve-ci-e2e-test resourceVersion: "21898" uid: 1134385d-5e64-47b7-b7f6-c78d2df09c77 kind: Event lastTimestamp: "2026-07-20T10:21:56Z" message: 'Failed to update status for InferenceService "success-200-isvc-64e78": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-64e78": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-20T10:21:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:21:56Z" name: success-200-isvc-64e78.18c3f853e0587de0 namespace: kserve-ci-e2e-test resourceVersion: "21931" uid: 8e577388-c97c-4a6f-9f48-d1ccdd1d474b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:21:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-64e78 namespace: kserve-ci-e2e-test resourceVersion: "21898" uid: 1134385d-5e64-47b7-b7f6-c78d2df09c77 kind: Event lastTimestamp: "2026-07-20T10:21:56Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-64e78": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-20T10:21:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:21:56Z" name: success-200-isvc-64e78.18c3f853e058c897 namespace: kserve-ci-e2e-test resourceVersion: "21932" uid: 758606ae-0960-4b7c-a945-0f587520520e reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:22:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-64e78 namespace: kserve-ci-e2e-test resourceVersion: "22459" uid: 1134385d-5e64-47b7-b7f6-c78d2df09c77 kind: Event lastTimestamp: "2026-07-20T10:22:45Z" message: InferenceService [success-200-isvc-64e78] is Ready metadata: creationTimestamp: "2026-07-20T10:22:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:22:45Z" name: success-200-isvc-64e78.18c3f85f3cfd9ebc namespace: kserve-ci-e2e-test resourceVersion: "22460" uid: 5d6a7105-92a7-42bc-8ef3-d99477afad26 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-20T10:31:22.066288Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6e423-predictor-6b546f494c-972tv namespace: kserve-ci-e2e-test resourceVersion: "26820" uid: 2a5b2dfd-f4b6-447f-b917-b8b38d2d6d3f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-6e423-predictor-6b546f494c-972tv to ip-10-0-143-155.ec2.internal metadata: creationTimestamp: "2026-07-20T10:31:22Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-20T10:31:22Z" name: success-200-isvc-6e423-predictor-6b546f494c-972tv.18c3f8d78eccd097 namespace: kserve-ci-e2e-test resourceVersion: "26826" uid: 6b63c72a-22de-4c56-a699-162b6688bebe reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5dc669d9-69hkz source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:22Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6e423-predictor-6b546f494c-972tv namespace: kserve-ci-e2e-test resourceVersion: "26824" uid: 2a5b2dfd-f4b6-447f-b917-b8b38d2d6d3f kind: Event lastTimestamp: "2026-07-20T10:31:22Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-6e423-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-20T10:31:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:31:22Z" name: success-200-isvc-6e423-predictor-6b546f494c-972tv.18c3f8d79b5a1c70 namespace: kserve-ci-e2e-test resourceVersion: "26863" uid: 2cc9e959-a6f9-437e-a282-90029d3ba4e4 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:23Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6e423-predictor-6b546f494c-972tv namespace: kserve-ci-e2e-test resourceVersion: "26830" uid: 2a5b2dfd-f4b6-447f-b917-b8b38d2d6d3f kind: Event lastTimestamp: "2026-07-20T10:31:23Z" message: Add eth0 [10.133.0.54/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-20T10:31:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-20T10:31:23Z" name: success-200-isvc-6e423-predictor-6b546f494c-972tv.18c3f8d7cc704a92 namespace: kserve-ci-e2e-test resourceVersion: "26877" uid: 6b84a5f8-de00-46ee-bcc8-9b7b577b730b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6e423-predictor-6b546f494c-972tv namespace: kserve-ci-e2e-test resourceVersion: "26824" uid: 2a5b2dfd-f4b6-447f-b917-b8b38d2d6d3f kind: Event lastTimestamp: "2026-07-20T10:31:23Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1771" already present on machine metadata: creationTimestamp: "2026-07-20T10:31:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:31:23Z" name: success-200-isvc-6e423-predictor-6b546f494c-972tv.18c3f8d7cdebe5a1 namespace: kserve-ci-e2e-test resourceVersion: "26879" uid: d0133e17-fd34-4168-9d1e-baf635e0fbf0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6e423-predictor-6b546f494c-972tv namespace: kserve-ci-e2e-test resourceVersion: "26824" uid: 2a5b2dfd-f4b6-447f-b917-b8b38d2d6d3f kind: Event lastTimestamp: "2026-07-20T10:31:23Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-20T10:31:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:31:23Z" name: success-200-isvc-6e423-predictor-6b546f494c-972tv.18c3f8d7d1f364b6 namespace: kserve-ci-e2e-test resourceVersion: "26880" uid: fb010d10-f614-4696-9f97-b4383e3bd290 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6e423-predictor-6b546f494c-972tv namespace: kserve-ci-e2e-test resourceVersion: "26824" uid: 2a5b2dfd-f4b6-447f-b917-b8b38d2d6d3f kind: Event lastTimestamp: "2026-07-20T10:31:23Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-20T10:31:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:31:23Z" name: success-200-isvc-6e423-predictor-6b546f494c-972tv.18c3f8d7d280b8cd namespace: kserve-ci-e2e-test resourceVersion: "26881" uid: 210a66f2-435e-4ca5-bfb0-5c759f055033 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6e423-predictor-6b546f494c-972tv namespace: kserve-ci-e2e-test resourceVersion: "26824" uid: 2a5b2dfd-f4b6-447f-b917-b8b38d2d6d3f kind: Event lastTimestamp: "2026-07-20T10:31:23Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-20T10:31:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:31:23Z" name: success-200-isvc-6e423-predictor-6b546f494c-972tv.18c3f8d7d28c0fa0 namespace: kserve-ci-e2e-test resourceVersion: "26882" uid: 1ce6ab62-aa19-4495-b87d-eef2d092aee5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6e423-predictor-6b546f494c-972tv namespace: kserve-ci-e2e-test resourceVersion: "26824" uid: 2a5b2dfd-f4b6-447f-b917-b8b38d2d6d3f kind: Event lastTimestamp: "2026-07-20T10:31:23Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-20T10:31:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:31:23Z" name: success-200-isvc-6e423-predictor-6b546f494c-972tv.18c3f8d7d6b0deda namespace: kserve-ci-e2e-test resourceVersion: "26887" uid: 34a0dc44-c3e6-4783-a4f7-8362f05fbcc0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6e423-predictor-6b546f494c-972tv namespace: kserve-ci-e2e-test resourceVersion: "26824" uid: 2a5b2dfd-f4b6-447f-b917-b8b38d2d6d3f kind: Event lastTimestamp: "2026-07-20T10:31:23Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:31:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:31:23Z" name: success-200-isvc-6e423-predictor-6b546f494c-972tv.18c3f8d7d721b068 namespace: kserve-ci-e2e-test resourceVersion: "26888" uid: 3763af19-9d11-4479-a364-d12e30c9a427 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-20T10:31:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6e423-predictor-6b546f494c-972tv namespace: kserve-ci-e2e-test resourceVersion: "26824" uid: 2a5b2dfd-f4b6-447f-b917-b8b38d2d6d3f kind: Event lastTimestamp: "2026-07-20T10:40:49Z" message: 'Readiness probe failed: dial tcp 10.133.0.54:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-20T10:31:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:40:49Z" name: success-200-isvc-6e423-predictor-6b546f494c-972tv.18c3f8d7fbc3ce66 namespace: kserve-ci-e2e-test resourceVersion: "31079" uid: ff8640d1-da90-4ee8-a782-0b9f3627f405 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:40:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6e423-predictor-6b546f494c-972tv namespace: kserve-ci-e2e-test resourceVersion: "26824" uid: 2a5b2dfd-f4b6-447f-b917-b8b38d2d6d3f kind: Event lastTimestamp: "2026-07-20T10:40:46Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-20T10:40:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:40:46Z" name: success-200-isvc-6e423-predictor-6b546f494c-972tv.18c3f95b116a35c7 namespace: kserve-ci-e2e-test resourceVersion: "31040" uid: 440f72eb-a82a-4e8d-9b35-0090c7944e41 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:40:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6e423-predictor-6b546f494c-972tv namespace: kserve-ci-e2e-test resourceVersion: "26824" uid: 2a5b2dfd-f4b6-447f-b917-b8b38d2d6d3f kind: Event lastTimestamp: "2026-07-20T10:40:46Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:40:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:40:46Z" name: success-200-isvc-6e423-predictor-6b546f494c-972tv.18c3f95b116ab7f3 namespace: kserve-ci-e2e-test resourceVersion: "31042" uid: f812e479-7ca9-4d69-979a-7e5a25081079 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:40:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6e423-predictor-6b546f494c-972tv namespace: kserve-ci-e2e-test resourceVersion: "26824" uid: 2a5b2dfd-f4b6-447f-b917-b8b38d2d6d3f kind: Event lastTimestamp: "2026-07-20T10:40:49Z" message: 'Readiness probe failed: Get "https://10.133.0.54:8643/healthz": dial tcp 10.133.0.54:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-20T10:40:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:40:49Z" name: success-200-isvc-6e423-predictor-6b546f494c-972tv.18c3f95bc458726a namespace: kserve-ci-e2e-test resourceVersion: "31078" uid: e904cb1d-e2bf-4b79-a716-2d5a1556819b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:22Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-6e423-predictor-6b546f494c namespace: kserve-ci-e2e-test resourceVersion: "26807" uid: b51f16e7-9e69-4479-a543-b64e72b1c299 kind: Event lastTimestamp: "2026-07-20T10:31:22Z" message: 'Created pod: success-200-isvc-6e423-predictor-6b546f494c-972tv' metadata: creationTimestamp: "2026-07-20T10:31:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:31:22Z" name: success-200-isvc-6e423-predictor-6b546f494c.18c3f8d78e5d5f96 namespace: kserve-ci-e2e-test resourceVersion: "26823" uid: 3e86c740-d3ad-4fd0-82a0-f73e8c7993b1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:22Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-6e423-predictor namespace: kserve-ci-e2e-test resourceVersion: "26800" uid: d8a61768-dd65-4099-b506-0f8d5a5bbb33 kind: Event lastTimestamp: "2026-07-20T10:31:22Z" message: Scaled up replica set success-200-isvc-6e423-predictor-6b546f494c from 0 to 1 metadata: creationTimestamp: "2026-07-20T10:31:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:31:22Z" name: success-200-isvc-6e423-predictor.18c3f8d78c7981b0 namespace: kserve-ci-e2e-test resourceVersion: "26810" uid: 7b2c8ee2-1ef7-463a-9872-331e6744ca42 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6e423-predictor namespace: kserve-ci-e2e-test resourceVersion: "26805" uid: 0b12ad58-7e8f-4dce-af96-d60a5e64f0d2 kind: Event lastTimestamp: "2026-07-20T10:31:37Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:31:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:31:37Z" name: success-200-isvc-6e423-predictor.18c3f8db0b5fbd78 namespace: kserve-ci-e2e-test resourceVersion: "27028" uid: e03d92b3-27c2-4533-976a-e22f232c5937 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6e423-predictor namespace: kserve-ci-e2e-test resourceVersion: "26805" uid: 0b12ad58-7e8f-4dce-af96-d60a5e64f0d2 kind: Event lastTimestamp: "2026-07-20T10:31:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:31:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:31:37Z" name: success-200-isvc-6e423-predictor.18c3f8db0bc33f2e namespace: kserve-ci-e2e-test resourceVersion: "27030" uid: c9c2fe06-889d-4c0a-ae55-3bba5c34c6e5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-20T10:31:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6e423-predictor namespace: kserve-ci-e2e-test resourceVersion: "27029" uid: 0b12ad58-7e8f-4dce-af96-d60a5e64f0d2 kind: Event lastTimestamp: "2026-07-20T10:32:52Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:31:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:32:52Z" name: success-200-isvc-6e423-predictor.18c3f8de8a83ec60 namespace: kserve-ci-e2e-test resourceVersion: "27726" uid: f5458a20-479d-40b6-9e7b-3898bb6ae68c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-20T10:31:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6e423-predictor namespace: kserve-ci-e2e-test resourceVersion: "27029" uid: 0b12ad58-7e8f-4dce-af96-d60a5e64f0d2 kind: Event lastTimestamp: "2026-07-20T10:32:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:31:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:32:52Z" name: success-200-isvc-6e423-predictor.18c3f8de8ae74327 namespace: kserve-ci-e2e-test resourceVersion: "27727" uid: 60656876-3cbb-4d64-a77d-6f0064ebcb6b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-20T10:31:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6e423 namespace: kserve-ci-e2e-test resourceVersion: "26779" uid: cf767a8c-4662-4d1b-acd6-704026694cb3 kind: Event lastTimestamp: "2026-07-20T10:32:20Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-20T10:31:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:32:20Z" name: success-200-isvc-6e423.18c3f8d78c8c709b namespace: kserve-ci-e2e-test resourceVersion: "27443" uid: aeec8993-0bc2-4eda-876a-a4bec5dd6b18 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6e423 namespace: kserve-ci-e2e-test resourceVersion: "26779" uid: cf767a8c-4662-4d1b-acd6-704026694cb3 kind: Event lastTimestamp: "2026-07-20T10:31:22Z" message: 'Failed to update status for InferenceService "success-200-isvc-6e423": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-6e423": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-20T10:31:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:31:22Z" name: success-200-isvc-6e423.18c3f8d78cf01c9e namespace: kserve-ci-e2e-test resourceVersion: "26815" uid: b2b5e1d2-24e3-4064-a39b-357e4fbfb022 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6e423 namespace: kserve-ci-e2e-test resourceVersion: "26779" uid: cf767a8c-4662-4d1b-acd6-704026694cb3 kind: Event lastTimestamp: "2026-07-20T10:31:22Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-6e423": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-20T10:31:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:31:22Z" name: success-200-isvc-6e423.18c3f8d78cf0d17a namespace: kserve-ci-e2e-test resourceVersion: "26817" uid: da402144-5ba3-44b9-ae4d-0ce5e2071819 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:32:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6e423 namespace: kserve-ci-e2e-test resourceVersion: "27440" uid: cf767a8c-4662-4d1b-acd6-704026694cb3 kind: Event lastTimestamp: "2026-07-20T10:32:19Z" message: InferenceService [success-200-isvc-6e423] is Ready metadata: creationTimestamp: "2026-07-20T10:32:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:32:19Z" name: success-200-isvc-6e423.18c3f8e50b1b5fe9 namespace: kserve-ci-e2e-test resourceVersion: "27441" uid: 66b6dad7-3b55-469f-b9df-31207df7c6cc reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-20T10:31:11.329292Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-78ab0-predictor-77ccfb98c9-mgczs namespace: kserve-ci-e2e-test resourceVersion: "26549" uid: 9d16fee0-3b7f-42a4-a8a1-52867747c7c0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-78ab0-predictor-77ccfb98c9-mgczs to ip-10-0-143-155.ec2.internal metadata: creationTimestamp: "2026-07-20T10:31:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-20T10:31:11Z" name: success-200-isvc-78ab0-predictor-77ccfb98c9-mgczs.18c3f8d50ed35f55 namespace: kserve-ci-e2e-test resourceVersion: "26553" uid: 82abf64a-ea12-4100-a386-b2de0665b6b9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5dc669d9-69hkz source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:11Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-78ab0-predictor-77ccfb98c9-mgczs namespace: kserve-ci-e2e-test resourceVersion: "26558" uid: 9d16fee0-3b7f-42a4-a8a1-52867747c7c0 kind: Event lastTimestamp: "2026-07-20T10:31:11Z" message: Add eth0 [10.133.0.52/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-20T10:31:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-20T10:31:11Z" name: success-200-isvc-78ab0-predictor-77ccfb98c9-mgczs.18c3f8d529dd0bdf namespace: kserve-ci-e2e-test resourceVersion: "26615" uid: 038cb0cc-b454-4407-ae33-b90a04fa5d02 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-78ab0-predictor-77ccfb98c9-mgczs namespace: kserve-ci-e2e-test resourceVersion: "26551" uid: 9d16fee0-3b7f-42a4-a8a1-52867747c7c0 kind: Event lastTimestamp: "2026-07-20T10:31:11Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1771" already present on machine metadata: creationTimestamp: "2026-07-20T10:31:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:31:11Z" name: success-200-isvc-78ab0-predictor-77ccfb98c9-mgczs.18c3f8d52ba2b545 namespace: kserve-ci-e2e-test resourceVersion: "26623" uid: 25f5a97c-db9d-4b31-b19d-4c23879e9df9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-78ab0-predictor-77ccfb98c9-mgczs namespace: kserve-ci-e2e-test resourceVersion: "26551" uid: 9d16fee0-3b7f-42a4-a8a1-52867747c7c0 kind: Event lastTimestamp: "2026-07-20T10:31:11Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-20T10:31:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:31:11Z" name: success-200-isvc-78ab0-predictor-77ccfb98c9-mgczs.18c3f8d52e7ab384 namespace: kserve-ci-e2e-test resourceVersion: "26629" uid: 4c6600e8-5d7e-4079-81b8-f051c2fbf574 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-78ab0-predictor-77ccfb98c9-mgczs namespace: kserve-ci-e2e-test resourceVersion: "26551" uid: 9d16fee0-3b7f-42a4-a8a1-52867747c7c0 kind: Event lastTimestamp: "2026-07-20T10:31:11Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-20T10:31:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:31:11Z" name: success-200-isvc-78ab0-predictor-77ccfb98c9-mgczs.18c3f8d52eeedbf8 namespace: kserve-ci-e2e-test resourceVersion: "26632" uid: f9cc7f93-3e78-4164-93f0-31c2cc21ddb3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-78ab0-predictor-77ccfb98c9-mgczs namespace: kserve-ci-e2e-test resourceVersion: "26551" uid: 9d16fee0-3b7f-42a4-a8a1-52867747c7c0 kind: Event lastTimestamp: "2026-07-20T10:31:11Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-20T10:31:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:31:11Z" name: success-200-isvc-78ab0-predictor-77ccfb98c9-mgczs.18c3f8d52ef7ce80 namespace: kserve-ci-e2e-test resourceVersion: "26634" uid: 0883229f-ba6c-4849-bfec-708858ca3e7b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-78ab0-predictor-77ccfb98c9-mgczs namespace: kserve-ci-e2e-test resourceVersion: "26551" uid: 9d16fee0-3b7f-42a4-a8a1-52867747c7c0 kind: Event lastTimestamp: "2026-07-20T10:31:11Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-20T10:31:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:31:11Z" name: success-200-isvc-78ab0-predictor-77ccfb98c9-mgczs.18c3f8d531b5da1e namespace: kserve-ci-e2e-test resourceVersion: "26637" uid: 341c2c71-29ad-4f3b-8422-5d1d20ea0636 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-78ab0-predictor-77ccfb98c9-mgczs namespace: kserve-ci-e2e-test resourceVersion: "26551" uid: 9d16fee0-3b7f-42a4-a8a1-52867747c7c0 kind: Event lastTimestamp: "2026-07-20T10:31:11Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:31:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:31:11Z" name: success-200-isvc-78ab0-predictor-77ccfb98c9-mgczs.18c3f8d532210143 namespace: kserve-ci-e2e-test resourceVersion: "26638" uid: 3a724573-59a3-4d2c-be7b-2f8d6bd918ea reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-20T10:31:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-78ab0-predictor-77ccfb98c9-mgczs namespace: kserve-ci-e2e-test resourceVersion: "26551" uid: 9d16fee0-3b7f-42a4-a8a1-52867747c7c0 kind: Event lastTimestamp: "2026-07-20T10:31:48Z" message: 'Readiness probe failed: dial tcp 10.133.0.52:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-20T10:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:31:48Z" name: success-200-isvc-78ab0-predictor-77ccfb98c9-mgczs.18c3f8d56810d114 namespace: kserve-ci-e2e-test resourceVersion: "27131" uid: 60284d9b-2402-4e02-8d25-c10f23f65a4c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:48:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-78ab0-predictor-77ccfb98c9-mgczs namespace: kserve-ci-e2e-test resourceVersion: "26551" uid: 9d16fee0-3b7f-42a4-a8a1-52867747c7c0 kind: Event lastTimestamp: "2026-07-20T10:48:30Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-20T10:48:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:48:30Z" name: success-200-isvc-78ab0-predictor-77ccfb98c9-mgczs.18c3f9c70e24c90d namespace: kserve-ci-e2e-test resourceVersion: "34268" uid: 071ee7e6-68c2-4a9f-aa11-2bb46b149468 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:48:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-78ab0-predictor-77ccfb98c9-mgczs namespace: kserve-ci-e2e-test resourceVersion: "26551" uid: 9d16fee0-3b7f-42a4-a8a1-52867747c7c0 kind: Event lastTimestamp: "2026-07-20T10:48:30Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:48:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:48:30Z" name: success-200-isvc-78ab0-predictor-77ccfb98c9-mgczs.18c3f9c70e254da2 namespace: kserve-ci-e2e-test resourceVersion: "34269" uid: 89977415-59ab-48e9-9aa1-77967ded2966 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-78ab0-predictor-77ccfb98c9 namespace: kserve-ci-e2e-test resourceVersion: "26540" uid: 06da1f33-09c8-4eb7-be6f-954db27f237f kind: Event lastTimestamp: "2026-07-20T10:31:11Z" message: 'Created pod: success-200-isvc-78ab0-predictor-77ccfb98c9-mgczs' metadata: creationTimestamp: "2026-07-20T10:31:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:31:11Z" name: success-200-isvc-78ab0-predictor-77ccfb98c9.18c3f8d50e57a7a9 namespace: kserve-ci-e2e-test resourceVersion: "26550" uid: 5e1b934a-7250-4f0b-9f50-5e0d04a8195e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-78ab0-predictor namespace: kserve-ci-e2e-test resourceVersion: "26536" uid: 058b9f39-5c68-4f88-b2a8-56c95e356827 kind: Event lastTimestamp: "2026-07-20T10:31:11Z" message: Scaled up replica set success-200-isvc-78ab0-predictor-77ccfb98c9 from 0 to 1 metadata: creationTimestamp: "2026-07-20T10:31:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:31:11Z" name: success-200-isvc-78ab0-predictor.18c3f8d50c8c7a9a namespace: kserve-ci-e2e-test resourceVersion: "26541" uid: d5c93514-b025-459e-95b1-0c003c5c0299 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-78ab0-predictor namespace: kserve-ci-e2e-test resourceVersion: "26539" uid: 149acdf2-c945-4404-b697-c0274eaee921 kind: Event lastTimestamp: "2026-07-20T10:31:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:31:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:31:26Z" name: success-200-isvc-78ab0-predictor.18c3f8d88b45486f namespace: kserve-ci-e2e-test resourceVersion: "26938" uid: a50c126b-0214-43a9-9425-9babf8cf8aca reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-78ab0-predictor namespace: kserve-ci-e2e-test resourceVersion: "26539" uid: 149acdf2-c945-4404-b697-c0274eaee921 kind: Event lastTimestamp: "2026-07-20T10:31:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:31:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:31:26Z" name: success-200-isvc-78ab0-predictor.18c3f8d88bab4095 namespace: kserve-ci-e2e-test resourceVersion: "26940" uid: cc4685e1-2454-4369-98ac-2cc378cdfe7f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-20T10:31:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-78ab0-predictor namespace: kserve-ci-e2e-test resourceVersion: "26939" uid: 149acdf2-c945-4404-b697-c0274eaee921 kind: Event lastTimestamp: "2026-07-20T10:32:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:31:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:32:26Z" name: success-200-isvc-78ab0-predictor.18c3f8dc0a89b420 namespace: kserve-ci-e2e-test resourceVersion: "27490" uid: b3038acb-fbf5-4f7b-aa6d-35694fbae242 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-20T10:31:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-78ab0-predictor namespace: kserve-ci-e2e-test resourceVersion: "26939" uid: 149acdf2-c945-4404-b697-c0274eaee921 kind: Event lastTimestamp: "2026-07-20T10:32:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:31:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:32:26Z" name: success-200-isvc-78ab0-predictor.18c3f8dc0af05d0a namespace: kserve-ci-e2e-test resourceVersion: "27491" uid: 0160eedc-aa3b-4ee2-a666-4c7b9dda428b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-20T10:31:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-78ab0 namespace: kserve-ci-e2e-test resourceVersion: "26513" uid: 57acaa69-1216-4c46-824e-f24219ea23cd kind: Event lastTimestamp: "2026-07-20T10:31:58Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-20T10:31:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:31:58Z" name: success-200-isvc-78ab0.18c3f8d50cb86734 namespace: kserve-ci-e2e-test resourceVersion: "27219" uid: 75e5e20e-003f-42d6-aac5-1b47cd74b52b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-78ab0 namespace: kserve-ci-e2e-test resourceVersion: "26513" uid: 57acaa69-1216-4c46-824e-f24219ea23cd kind: Event lastTimestamp: "2026-07-20T10:31:11Z" message: 'Failed to update status for InferenceService "success-200-isvc-78ab0": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-78ab0": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-20T10:31:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:31:11Z" name: success-200-isvc-78ab0.18c3f8d50d200f7b namespace: kserve-ci-e2e-test resourceVersion: "26545" uid: c585512b-11d1-4df3-b9c6-4e20387495b4 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-78ab0 namespace: kserve-ci-e2e-test resourceVersion: "26513" uid: 57acaa69-1216-4c46-824e-f24219ea23cd kind: Event lastTimestamp: "2026-07-20T10:31:11Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-78ab0": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-20T10:31:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:31:11Z" name: success-200-isvc-78ab0.18c3f8d50d2042bf namespace: kserve-ci-e2e-test resourceVersion: "26547" uid: 289c6a5f-c662-4784-9866-424d2fa65ffd reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-78ab0 namespace: kserve-ci-e2e-test resourceVersion: "27217" uid: 57acaa69-1216-4c46-824e-f24219ea23cd kind: Event lastTimestamp: "2026-07-20T10:31:58Z" message: InferenceService [success-200-isvc-78ab0] is Ready metadata: creationTimestamp: "2026-07-20T10:31:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:31:58Z" name: success-200-isvc-78ab0.18c3f8e0242483e7 namespace: kserve-ci-e2e-test resourceVersion: "27218" uid: ca0e0c3f-4056-4f28-bede-11e07c3237b7 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-20T10:11:21.422700Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-78f99-predictor-676885594f-6drz2 namespace: kserve-ci-e2e-test resourceVersion: "16550" uid: 49cb0401-f5f7-429a-849a-e1f1953bf116 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-78f99-predictor-676885594f-6drz2 to ip-10-0-143-155.ec2.internal metadata: creationTimestamp: "2026-07-20T10:11:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-20T10:11:21Z" name: success-200-isvc-78f99-predictor-676885594f-6drz2.18c3f7c002dd9513 namespace: kserve-ci-e2e-test resourceVersion: "16567" uid: 26977d65-aa16-48b0-b01d-83909d6b6b1f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5dc669d9-69hkz source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:21Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-78f99-predictor-676885594f-6drz2 namespace: kserve-ci-e2e-test resourceVersion: "16559" uid: 49cb0401-f5f7-429a-849a-e1f1953bf116 kind: Event lastTimestamp: "2026-07-20T10:11:21Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-78f99-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-20T10:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:11:21Z" name: success-200-isvc-78f99-predictor-676885594f-6drz2.18c3f7c011edaeb7 namespace: kserve-ci-e2e-test resourceVersion: "16628" uid: 5ba88a55-765f-4fb3-a87d-4c0c7a11e23e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:22Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-78f99-predictor-676885594f-6drz2 namespace: kserve-ci-e2e-test resourceVersion: "16575" uid: 49cb0401-f5f7-429a-849a-e1f1953bf116 kind: Event lastTimestamp: "2026-07-20T10:11:22Z" message: Add eth0 [10.133.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-20T10:11:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-20T10:11:22Z" name: success-200-isvc-78f99-predictor-676885594f-6drz2.18c3f7c040869551 namespace: kserve-ci-e2e-test resourceVersion: "16649" uid: e1029867-8af0-4084-9242-d91a50379467 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-78f99-predictor-676885594f-6drz2 namespace: kserve-ci-e2e-test resourceVersion: "16559" uid: 49cb0401-f5f7-429a-849a-e1f1953bf116 kind: Event lastTimestamp: "2026-07-20T10:11:22Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1771" already present on machine metadata: creationTimestamp: "2026-07-20T10:11:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:11:22Z" name: success-200-isvc-78f99-predictor-676885594f-6drz2.18c3f7c042282ca5 namespace: kserve-ci-e2e-test resourceVersion: "16651" uid: aaa92257-8e50-42f9-b370-d9a06851628e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-78f99-predictor-676885594f-6drz2 namespace: kserve-ci-e2e-test resourceVersion: "16559" uid: 49cb0401-f5f7-429a-849a-e1f1953bf116 kind: Event lastTimestamp: "2026-07-20T10:11:22Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-20T10:11:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:11:22Z" name: success-200-isvc-78f99-predictor-676885594f-6drz2.18c3f7c044ce4c17 namespace: kserve-ci-e2e-test resourceVersion: "16652" uid: 34f85e01-84bb-41f4-92eb-65aebfe42463 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-78f99-predictor-676885594f-6drz2 namespace: kserve-ci-e2e-test resourceVersion: "16559" uid: 49cb0401-f5f7-429a-849a-e1f1953bf116 kind: Event lastTimestamp: "2026-07-20T10:11:22Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-20T10:11:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:11:22Z" name: success-200-isvc-78f99-predictor-676885594f-6drz2.18c3f7c04530e339 namespace: kserve-ci-e2e-test resourceVersion: "16653" uid: ae1daaca-9785-42ac-8fb5-b00e60381fc1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-78f99-predictor-676885594f-6drz2 namespace: kserve-ci-e2e-test resourceVersion: "16559" uid: 49cb0401-f5f7-429a-849a-e1f1953bf116 kind: Event lastTimestamp: "2026-07-20T10:11:22Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-20T10:11:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:11:22Z" name: success-200-isvc-78f99-predictor-676885594f-6drz2.18c3f7c04539f5ea namespace: kserve-ci-e2e-test resourceVersion: "16654" uid: 1e7f0edb-cbcf-4e52-9bcf-c77c40e4cc49 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-78f99-predictor-676885594f-6drz2 namespace: kserve-ci-e2e-test resourceVersion: "16559" uid: 49cb0401-f5f7-429a-849a-e1f1953bf116 kind: Event lastTimestamp: "2026-07-20T10:11:22Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-20T10:11:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:11:22Z" name: success-200-isvc-78f99-predictor-676885594f-6drz2.18c3f7c0482b2934 namespace: kserve-ci-e2e-test resourceVersion: "16655" uid: 0aadd012-5b54-4a3b-ad14-c04d45ffdb3a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-78f99-predictor-676885594f-6drz2 namespace: kserve-ci-e2e-test resourceVersion: "16559" uid: 49cb0401-f5f7-429a-849a-e1f1953bf116 kind: Event lastTimestamp: "2026-07-20T10:11:22Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:11:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:11:22Z" name: success-200-isvc-78f99-predictor-676885594f-6drz2.18c3f7c04896fa84 namespace: kserve-ci-e2e-test resourceVersion: "16656" uid: 80930319-4b1a-48c1-8ee0-de2d53df5eab reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-20T10:11:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-78f99-predictor-676885594f-6drz2 namespace: kserve-ci-e2e-test resourceVersion: "16559" uid: 49cb0401-f5f7-429a-849a-e1f1953bf116 kind: Event lastTimestamp: "2026-07-20T10:20:39Z" message: 'Readiness probe failed: dial tcp 10.133.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-20T10:11:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:20:39Z" name: success-200-isvc-78f99-predictor-676885594f-6drz2.18c3f7c0614faa9d namespace: kserve-ci-e2e-test resourceVersion: "20981" uid: 4c919f1c-4eb0-4319-a80d-6a1dbc0d046c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-78f99-predictor-676885594f-6drz2 namespace: kserve-ci-e2e-test resourceVersion: "16559" uid: 49cb0401-f5f7-429a-849a-e1f1953bf116 kind: Event lastTimestamp: "2026-07-20T10:20:36Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-20T10:20:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:20:36Z" name: success-200-isvc-78f99-predictor-676885594f-6drz2.18c3f8412a575d8b namespace: kserve-ci-e2e-test resourceVersion: "20850" uid: 9ad06846-bb07-4897-9480-f3280d658caf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-78f99-predictor-676885594f-6drz2 namespace: kserve-ci-e2e-test resourceVersion: "16559" uid: 49cb0401-f5f7-429a-849a-e1f1953bf116 kind: Event lastTimestamp: "2026-07-20T10:20:36Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:20:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:20:36Z" name: success-200-isvc-78f99-predictor-676885594f-6drz2.18c3f8412a580364 namespace: kserve-ci-e2e-test resourceVersion: "20852" uid: 5eb0dc33-586e-446e-a512-a7001b0eb29a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:20:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-78f99-predictor-676885594f-6drz2 namespace: kserve-ci-e2e-test resourceVersion: "16559" uid: 49cb0401-f5f7-429a-849a-e1f1953bf116 kind: Event lastTimestamp: "2026-07-20T10:20:39Z" message: 'Readiness probe failed: Get "https://10.133.0.40:8643/healthz": dial tcp 10.133.0.40:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-20T10:20:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:20:39Z" name: success-200-isvc-78f99-predictor-676885594f-6drz2.18c3f841d5d0f7a8 namespace: kserve-ci-e2e-test resourceVersion: "20979" uid: 67d61a3c-da71-48b0-bf89-aff00f74f853 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-78f99-predictor-676885594f namespace: kserve-ci-e2e-test resourceVersion: "16537" uid: 6910b948-1fb5-4130-a8c9-fe07fa4447e5 kind: Event lastTimestamp: "2026-07-20T10:11:21Z" message: 'Created pod: success-200-isvc-78f99-predictor-676885594f-6drz2' metadata: creationTimestamp: "2026-07-20T10:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:11:21Z" name: success-200-isvc-78f99-predictor-676885594f.18c3f7c00216360c namespace: kserve-ci-e2e-test resourceVersion: "16558" uid: ed6098d5-a3ee-4fa7-8f81-9fc4d7620835 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-78f99-predictor namespace: kserve-ci-e2e-test resourceVersion: "16529" uid: 334ffb07-8ed5-4687-9d12-564f0093a7b4 kind: Event lastTimestamp: "2026-07-20T10:11:21Z" message: Scaled up replica set success-200-isvc-78f99-predictor-676885594f from 0 to 1 metadata: creationTimestamp: "2026-07-20T10:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:11:21Z" name: success-200-isvc-78f99-predictor.18c3f7bfffb55cd8 namespace: kserve-ci-e2e-test resourceVersion: "16542" uid: 49c9bbcc-51f2-45ec-afa8-c2ebf33b591f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-20T10:11:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-78f99-predictor namespace: kserve-ci-e2e-test resourceVersion: "16534" uid: e21c07cb-41dd-4311-964a-e3b4b27eefe7 kind: Event lastTimestamp: "2026-07-20T10:11:51Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:11:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:11:51Z" name: success-200-isvc-78f99-predictor.18c3f7c37e2877c7 namespace: kserve-ci-e2e-test resourceVersion: "16947" uid: 96e395df-e992-4dab-99e0-d53d110d1070 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-20T10:11:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-78f99-predictor namespace: kserve-ci-e2e-test resourceVersion: "16534" uid: e21c07cb-41dd-4311-964a-e3b4b27eefe7 kind: Event lastTimestamp: "2026-07-20T10:11:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:11:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:11:51Z" name: success-200-isvc-78f99-predictor.18c3f7c37e9437ab namespace: kserve-ci-e2e-test resourceVersion: "16948" uid: 9d0ddeb2-1d1f-4abd-9275-26bb9c4107d3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-20T10:12:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-78f99-predictor namespace: kserve-ci-e2e-test resourceVersion: "16820" uid: e21c07cb-41dd-4311-964a-e3b4b27eefe7 kind: Event lastTimestamp: "2026-07-20T10:12:36Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:12:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:12:36Z" name: success-200-isvc-78f99-predictor.18c3f7ca7c74796d namespace: kserve-ci-e2e-test resourceVersion: "17333" uid: 6ddc2e36-8076-4dac-a1a9-46eb5d0b21c2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-20T10:12:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-78f99-predictor namespace: kserve-ci-e2e-test resourceVersion: "16820" uid: e21c07cb-41dd-4311-964a-e3b4b27eefe7 kind: Event lastTimestamp: "2026-07-20T10:12:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:12:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:12:36Z" name: success-200-isvc-78f99-predictor.18c3f7ca7cd6b7e2 namespace: kserve-ci-e2e-test resourceVersion: "17334" uid: ab875d57-5ac9-4ef6-8010-d97461ef587d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-20T10:11:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-78f99 namespace: kserve-ci-e2e-test resourceVersion: "16521" uid: 5a9924cc-7d29-4eca-8567-bb51f0c3e1c3 kind: Event lastTimestamp: "2026-07-20T10:12:09Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-20T10:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:12:09Z" name: success-200-isvc-78f99.18c3f7bfff58aca5 namespace: kserve-ci-e2e-test resourceVersion: "17100" uid: ea79894b-322d-491d-9c9e-f92af843266a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:12:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-78f99 namespace: kserve-ci-e2e-test resourceVersion: "17097" uid: 5a9924cc-7d29-4eca-8567-bb51f0c3e1c3 kind: Event lastTimestamp: "2026-07-20T10:12:09Z" message: InferenceService [success-200-isvc-78f99] is Ready metadata: creationTimestamp: "2026-07-20T10:12:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:12:09Z" name: success-200-isvc-78f99.18c3f7cb1f729b53 namespace: kserve-ci-e2e-test resourceVersion: "17098" uid: 2ac74373-6375-4c4b-bba8-82794035dc7f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-20T10:10:31.411680Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7f819-predictor-6c79894b85-nc5nt namespace: kserve-ci-e2e-test resourceVersion: "15944" uid: 96d89d43-57c5-4392-8a8a-9095864d32e8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-7f819-predictor-6c79894b85-nc5nt to ip-10-0-143-155.ec2.internal metadata: creationTimestamp: "2026-07-20T10:10:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-20T10:10:31Z" name: success-200-isvc-7f819-predictor-6c79894b85-nc5nt.18c3f7b45dfa034d namespace: kserve-ci-e2e-test resourceVersion: "15949" uid: 2cae8b6e-0983-48c4-90ff-4ce5216f7ffd reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5dc669d9-69hkz source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:31Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7f819-predictor-6c79894b85-nc5nt namespace: kserve-ci-e2e-test resourceVersion: "15956" uid: 96d89d43-57c5-4392-8a8a-9095864d32e8 kind: Event lastTimestamp: "2026-07-20T10:10:31Z" message: Add eth0 [10.133.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-20T10:10:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-20T10:10:31Z" name: success-200-isvc-7f819-predictor-6c79894b85-nc5nt.18c3f7b47792b245 namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: 07253f0c-fc81-4715-8d19-f4b4266398ba reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7f819-predictor-6c79894b85-nc5nt namespace: kserve-ci-e2e-test resourceVersion: "15947" uid: 96d89d43-57c5-4392-8a8a-9095864d32e8 kind: Event lastTimestamp: "2026-07-20T10:10:31Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1771" already present on machine metadata: creationTimestamp: "2026-07-20T10:10:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:10:31Z" name: success-200-isvc-7f819-predictor-6c79894b85-nc5nt.18c3f7b47937cf97 namespace: kserve-ci-e2e-test resourceVersion: "15996" uid: 2088438d-8f89-4616-a08f-8632791af1a4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7f819-predictor-6c79894b85-nc5nt namespace: kserve-ci-e2e-test resourceVersion: "15947" uid: 96d89d43-57c5-4392-8a8a-9095864d32e8 kind: Event lastTimestamp: "2026-07-20T10:10:31Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-20T10:10:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:10:31Z" name: success-200-isvc-7f819-predictor-6c79894b85-nc5nt.18c3f7b47c0c3d4b namespace: kserve-ci-e2e-test resourceVersion: "15997" uid: 5f2067bf-8d9b-43bb-9b44-f30bcb2f5d1d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7f819-predictor-6c79894b85-nc5nt namespace: kserve-ci-e2e-test resourceVersion: "15947" uid: 96d89d43-57c5-4392-8a8a-9095864d32e8 kind: Event lastTimestamp: "2026-07-20T10:10:31Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-20T10:10:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:10:31Z" name: success-200-isvc-7f819-predictor-6c79894b85-nc5nt.18c3f7b47c6cc0e0 namespace: kserve-ci-e2e-test resourceVersion: "15998" uid: 9115dac7-358f-4fba-96a2-22bbdda7c550 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7f819-predictor-6c79894b85-nc5nt namespace: kserve-ci-e2e-test resourceVersion: "15947" uid: 96d89d43-57c5-4392-8a8a-9095864d32e8 kind: Event lastTimestamp: "2026-07-20T10:10:31Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-20T10:10:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:10:31Z" name: success-200-isvc-7f819-predictor-6c79894b85-nc5nt.18c3f7b47c75dd92 namespace: kserve-ci-e2e-test resourceVersion: "15999" uid: 155ea438-eacd-4ee3-9574-b4ff08bc2ffe reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7f819-predictor-6c79894b85-nc5nt namespace: kserve-ci-e2e-test resourceVersion: "15947" uid: 96d89d43-57c5-4392-8a8a-9095864d32e8 kind: Event lastTimestamp: "2026-07-20T10:10:31Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-20T10:10:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:10:31Z" name: success-200-isvc-7f819-predictor-6c79894b85-nc5nt.18c3f7b47f39b323 namespace: kserve-ci-e2e-test resourceVersion: "16000" uid: b2af27b8-a9ba-4b4d-ae9f-abaf370a8464 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7f819-predictor-6c79894b85-nc5nt namespace: kserve-ci-e2e-test resourceVersion: "15947" uid: 96d89d43-57c5-4392-8a8a-9095864d32e8 kind: Event lastTimestamp: "2026-07-20T10:10:31Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:10:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:10:31Z" name: success-200-isvc-7f819-predictor-6c79894b85-nc5nt.18c3f7b47f9814e4 namespace: kserve-ci-e2e-test resourceVersion: "16002" uid: 1c236fe6-dd9c-4e17-a347-e0b51d52631b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-20T10:10:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7f819-predictor-6c79894b85-nc5nt namespace: kserve-ci-e2e-test resourceVersion: "15947" uid: 96d89d43-57c5-4392-8a8a-9095864d32e8 kind: Event lastTimestamp: "2026-07-20T10:19:49Z" message: 'Readiness probe failed: dial tcp 10.133.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-20T10:10:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:19:49Z" name: success-200-isvc-7f819-predictor-6c79894b85-nc5nt.18c3f7b4ea3dab11 namespace: kserve-ci-e2e-test resourceVersion: "20446" uid: 31b2a4c8-233d-4070-b564-bb10d80e5907 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:19:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7f819-predictor-6c79894b85-nc5nt namespace: kserve-ci-e2e-test resourceVersion: "15947" uid: 96d89d43-57c5-4392-8a8a-9095864d32e8 kind: Event lastTimestamp: "2026-07-20T10:19:46Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-20T10:19:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:19:46Z" name: success-200-isvc-7f819-predictor-6c79894b85-nc5nt.18c3f835a7f807ee namespace: kserve-ci-e2e-test resourceVersion: "20363" uid: aa1d9515-6d5f-434f-9358-21d04d12d1b9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:19:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7f819-predictor-6c79894b85-nc5nt namespace: kserve-ci-e2e-test resourceVersion: "15947" uid: 96d89d43-57c5-4392-8a8a-9095864d32e8 kind: Event lastTimestamp: "2026-07-20T10:19:46Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:19:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:19:46Z" name: success-200-isvc-7f819-predictor-6c79894b85-nc5nt.18c3f835a7f87ef2 namespace: kserve-ci-e2e-test resourceVersion: "20366" uid: 87400dac-e55b-4ff3-b2d0-d3cac1e028f0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:19:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7f819-predictor-6c79894b85-nc5nt namespace: kserve-ci-e2e-test resourceVersion: "15947" uid: 96d89d43-57c5-4392-8a8a-9095864d32e8 kind: Event lastTimestamp: "2026-07-20T10:19:49Z" message: 'Readiness probe failed: Get "https://10.133.0.38:8643/healthz": dial tcp 10.133.0.38:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-20T10:19:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:19:49Z" name: success-200-isvc-7f819-predictor-6c79894b85-nc5nt.18c3f8365ecb1618 namespace: kserve-ci-e2e-test resourceVersion: "20445" uid: 8705b27d-469e-4e77-a3a6-cdc486bdf7f7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-7f819-predictor-6c79894b85 namespace: kserve-ci-e2e-test resourceVersion: "15936" uid: f3996f81-e743-4039-981c-971d3fc7a21d kind: Event lastTimestamp: "2026-07-20T10:10:31Z" message: 'Created pod: success-200-isvc-7f819-predictor-6c79894b85-nc5nt' metadata: creationTimestamp: "2026-07-20T10:10:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:10:31Z" name: success-200-isvc-7f819-predictor-6c79894b85.18c3f7b45d8f23a0 namespace: kserve-ci-e2e-test resourceVersion: "15946" uid: c2a42f91-170a-48e8-91c1-636b05d9ca1c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-7f819-predictor namespace: kserve-ci-e2e-test resourceVersion: "15931" uid: b434aa2c-8c91-4f6f-9dfb-f0a8add868ab kind: Event lastTimestamp: "2026-07-20T10:10:31Z" message: Scaled up replica set success-200-isvc-7f819-predictor-6c79894b85 from 0 to 1 metadata: creationTimestamp: "2026-07-20T10:10:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:10:31Z" name: success-200-isvc-7f819-predictor.18c3f7b45bee02c3 namespace: kserve-ci-e2e-test resourceVersion: "15938" uid: 4ca3a78a-a869-4051-9d0f-c6093122b952 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-20T10:10:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7f819-predictor namespace: kserve-ci-e2e-test resourceVersion: "15934" uid: ed406e8d-9771-4ff7-a284-3900ac75233d kind: Event lastTimestamp: "2026-07-20T10:11:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:10:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:11:01Z" name: success-200-isvc-7f819-predictor.18c3f7b7db1ec01c namespace: kserve-ci-e2e-test resourceVersion: "16279" uid: 0f15b51f-7046-4e5f-bdde-1a39d3b86e47 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-20T10:10:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7f819-predictor namespace: kserve-ci-e2e-test resourceVersion: "15934" uid: ed406e8d-9771-4ff7-a284-3900ac75233d kind: Event lastTimestamp: "2026-07-20T10:11:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:10:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:11:01Z" name: success-200-isvc-7f819-predictor.18c3f7b7dbeb933d namespace: kserve-ci-e2e-test resourceVersion: "16280" uid: 6f2b6816-7a17-44aa-9955-08403876cc9d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-20T10:11:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7f819-predictor namespace: kserve-ci-e2e-test resourceVersion: "16140" uid: ed406e8d-9771-4ff7-a284-3900ac75233d kind: Event lastTimestamp: "2026-07-20T10:11:31Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:11:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:11:31Z" name: success-200-isvc-7f819-predictor.18c3f7bed9b19ac8 namespace: kserve-ci-e2e-test resourceVersion: "16742" uid: 8dce60ee-f949-4f38-9010-68300563315b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-20T10:11:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7f819-predictor namespace: kserve-ci-e2e-test resourceVersion: "16140" uid: ed406e8d-9771-4ff7-a284-3900ac75233d kind: Event lastTimestamp: "2026-07-20T10:11:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:11:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:11:31Z" name: success-200-isvc-7f819-predictor.18c3f7beda18a8b3 namespace: kserve-ci-e2e-test resourceVersion: "16743" uid: 37844e67-8469-40bb-8048-d4bac931aaf6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-20T10:10:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7f819 namespace: kserve-ci-e2e-test resourceVersion: "15910" uid: 39d11ed1-a1a2-4c5d-899f-6321d88ca7dc kind: Event lastTimestamp: "2026-07-20T10:11:19Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-20T10:10:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:11:19Z" name: success-200-isvc-7f819.18c3f7b45c13ea82 namespace: kserve-ci-e2e-test resourceVersion: "16492" uid: 9e796da2-f63e-4f63-959d-0d8bd78a4cd1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7f819 namespace: kserve-ci-e2e-test resourceVersion: "15910" uid: 39d11ed1-a1a2-4c5d-899f-6321d88ca7dc kind: Event lastTimestamp: "2026-07-20T10:10:31Z" message: 'Failed to update status for InferenceService "success-200-isvc-7f819": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-7f819": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-20T10:10:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:10:31Z" name: success-200-isvc-7f819.18c3f7b45c75db4b namespace: kserve-ci-e2e-test resourceVersion: "15941" uid: 9f32bdc0-4edb-4afc-ac6e-535047f4d023 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7f819 namespace: kserve-ci-e2e-test resourceVersion: "15910" uid: 39d11ed1-a1a2-4c5d-899f-6321d88ca7dc kind: Event lastTimestamp: "2026-07-20T10:10:31Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-7f819": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-20T10:10:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:10:31Z" name: success-200-isvc-7f819.18c3f7b45c76403d namespace: kserve-ci-e2e-test resourceVersion: "15943" uid: 30b0188f-2fc3-4901-b3ac-d922b5aee5b4 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7f819 namespace: kserve-ci-e2e-test resourceVersion: "16489" uid: 39d11ed1-a1a2-4c5d-899f-6321d88ca7dc kind: Event lastTimestamp: "2026-07-20T10:11:19Z" message: InferenceService [success-200-isvc-7f819] is Ready metadata: creationTimestamp: "2026-07-20T10:11:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:11:19Z" name: success-200-isvc-7f819.18c3f7bfa8a43211 namespace: kserve-ci-e2e-test resourceVersion: "16490" uid: 955d7274-c692-43c3-9373-eccb06980a82 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-20T10:30:11.765027Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a956e-predictor-6f7cf8c9-8n552 namespace: kserve-ci-e2e-test resourceVersion: "25902" uid: 002f075b-ee91-4ce2-b135-2704a2312877 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-a956e-predictor-6f7cf8c9-8n552 to ip-10-0-143-155.ec2.internal metadata: creationTimestamp: "2026-07-20T10:30:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-20T10:30:11Z" name: success-200-isvc-a956e-predictor-6f7cf8c9-8n552.18c3f8c73084df9f namespace: kserve-ci-e2e-test resourceVersion: "25919" uid: 45916a3e-a9fb-4e55-8bfd-09c44b3d1124 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5dc669d9-69hkz source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:30:12Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a956e-predictor-6f7cf8c9-8n552 namespace: kserve-ci-e2e-test resourceVersion: "25933" uid: 002f075b-ee91-4ce2-b135-2704a2312877 kind: Event lastTimestamp: "2026-07-20T10:30:12Z" message: Add eth0 [10.133.0.50/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-20T10:30:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-20T10:30:12Z" name: success-200-isvc-a956e-predictor-6f7cf8c9-8n552.18c3f8c74aab37c0 namespace: kserve-ci-e2e-test resourceVersion: "25979" uid: 79205b68-154c-4198-82b4-a8fbb1eb401a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:30:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a956e-predictor-6f7cf8c9-8n552 namespace: kserve-ci-e2e-test resourceVersion: "25909" uid: 002f075b-ee91-4ce2-b135-2704a2312877 kind: Event lastTimestamp: "2026-07-20T10:30:12Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1771" already present on machine metadata: creationTimestamp: "2026-07-20T10:30:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:30:12Z" name: success-200-isvc-a956e-predictor-6f7cf8c9-8n552.18c3f8c74c3225a9 namespace: kserve-ci-e2e-test resourceVersion: "25981" uid: de028fb9-76ea-4de7-a98f-291ea313b59a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:30:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a956e-predictor-6f7cf8c9-8n552 namespace: kserve-ci-e2e-test resourceVersion: "25909" uid: 002f075b-ee91-4ce2-b135-2704a2312877 kind: Event lastTimestamp: "2026-07-20T10:30:12Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-20T10:30:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:30:12Z" name: success-200-isvc-a956e-predictor-6f7cf8c9-8n552.18c3f8c74f6f9f2c namespace: kserve-ci-e2e-test resourceVersion: "25982" uid: c78fb72c-dcd1-402e-bf2c-4b2b90123b6f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:30:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a956e-predictor-6f7cf8c9-8n552 namespace: kserve-ci-e2e-test resourceVersion: "25909" uid: 002f075b-ee91-4ce2-b135-2704a2312877 kind: Event lastTimestamp: "2026-07-20T10:30:12Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-20T10:30:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:30:12Z" name: success-200-isvc-a956e-predictor-6f7cf8c9-8n552.18c3f8c74fe64c45 namespace: kserve-ci-e2e-test resourceVersion: "25983" uid: 7f7c3efb-89ea-4860-a3ff-703d0dd910d7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:30:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a956e-predictor-6f7cf8c9-8n552 namespace: kserve-ci-e2e-test resourceVersion: "25909" uid: 002f075b-ee91-4ce2-b135-2704a2312877 kind: Event lastTimestamp: "2026-07-20T10:30:12Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-20T10:30:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:30:12Z" name: success-200-isvc-a956e-predictor-6f7cf8c9-8n552.18c3f8c74ff13125 namespace: kserve-ci-e2e-test resourceVersion: "25984" uid: c52bc6e5-4e05-40cd-92e8-32f2778f349c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:30:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a956e-predictor-6f7cf8c9-8n552 namespace: kserve-ci-e2e-test resourceVersion: "25909" uid: 002f075b-ee91-4ce2-b135-2704a2312877 kind: Event lastTimestamp: "2026-07-20T10:30:12Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-20T10:30:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:30:12Z" name: success-200-isvc-a956e-predictor-6f7cf8c9-8n552.18c3f8c7542e21b4 namespace: kserve-ci-e2e-test resourceVersion: "25988" uid: e42d13d7-caf3-40e9-9c93-6d73e87eb4ef reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:30:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a956e-predictor-6f7cf8c9-8n552 namespace: kserve-ci-e2e-test resourceVersion: "25909" uid: 002f075b-ee91-4ce2-b135-2704a2312877 kind: Event lastTimestamp: "2026-07-20T10:30:12Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:30:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:30:12Z" name: success-200-isvc-a956e-predictor-6f7cf8c9-8n552.18c3f8c754a3609d namespace: kserve-ci-e2e-test resourceVersion: "25989" uid: e6fc3e1d-5bfb-41ff-951b-9a31fb370c7d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-20T10:30:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a956e-predictor-6f7cf8c9-8n552 namespace: kserve-ci-e2e-test resourceVersion: "25909" uid: 002f075b-ee91-4ce2-b135-2704a2312877 kind: Event lastTimestamp: "2026-07-20T10:30:49Z" message: 'Readiness probe failed: dial tcp 10.133.0.50:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-20T10:30:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:30:49Z" name: success-200-isvc-a956e-predictor-6f7cf8c9-8n552.18c3f8c7998e4073 namespace: kserve-ci-e2e-test resourceVersion: "26308" uid: 5fe9a3eb-da86-402d-8cdd-48a3fb200a39 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a956e-predictor-6f7cf8c9-8n552 namespace: kserve-ci-e2e-test resourceVersion: "25909" uid: 002f075b-ee91-4ce2-b135-2704a2312877 kind: Event lastTimestamp: "2026-07-20T10:31:21Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-20T10:31:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:31:21Z" name: success-200-isvc-a956e-predictor-6f7cf8c9-8n552.18c3f8d78a413dfd namespace: kserve-ci-e2e-test resourceVersion: "26788" uid: 9f399e86-901c-42a7-af62-697cdfe305ad reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a956e-predictor-6f7cf8c9-8n552 namespace: kserve-ci-e2e-test resourceVersion: "25909" uid: 002f075b-ee91-4ce2-b135-2704a2312877 kind: Event lastTimestamp: "2026-07-20T10:31:21Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:31:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:31:21Z" name: success-200-isvc-a956e-predictor-6f7cf8c9-8n552.18c3f8d78a420b39 namespace: kserve-ci-e2e-test resourceVersion: "26791" uid: a50a2843-dfe8-4c02-b8e7-000754464e26 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:31:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a956e-predictor-6f7cf8c9-8n552 namespace: kserve-ci-e2e-test resourceVersion: "25909" uid: 002f075b-ee91-4ce2-b135-2704a2312877 kind: Event lastTimestamp: "2026-07-20T10:31:24Z" message: 'Readiness probe failed: Get "https://10.133.0.50:8643/healthz": dial tcp 10.133.0.50:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-20T10:31:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:31:24Z" name: success-200-isvc-a956e-predictor-6f7cf8c9-8n552.18c3f8d821bf0766 namespace: kserve-ci-e2e-test resourceVersion: "26914" uid: e5860f38-e1e0-4106-aa6c-cfa97291cef2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:30:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-a956e-predictor-6f7cf8c9 namespace: kserve-ci-e2e-test resourceVersion: "25892" uid: 135778a2-1c9e-4fb9-ba55-89c033ceabff kind: Event lastTimestamp: "2026-07-20T10:30:11Z" message: 'Created pod: success-200-isvc-a956e-predictor-6f7cf8c9-8n552' metadata: creationTimestamp: "2026-07-20T10:30:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:30:11Z" name: success-200-isvc-a956e-predictor-6f7cf8c9.18c3f8c72fdb58ef namespace: kserve-ci-e2e-test resourceVersion: "25910" uid: 7823bfc8-1471-4eeb-bdd0-317aa486f61c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:30:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-a956e-predictor namespace: kserve-ci-e2e-test resourceVersion: "25888" uid: bb416fce-99b4-46c3-9f55-f7a4834cc8b5 kind: Event lastTimestamp: "2026-07-20T10:30:11Z" message: Scaled up replica set success-200-isvc-a956e-predictor-6f7cf8c9 from 0 to 1 metadata: creationTimestamp: "2026-07-20T10:30:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:30:11Z" name: success-200-isvc-a956e-predictor.18c3f8c72c1e55a7 namespace: kserve-ci-e2e-test resourceVersion: "25895" uid: 1e30a507-2265-4730-bb9e-615e86ebd1d7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-20T10:30:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a956e-predictor namespace: kserve-ci-e2e-test resourceVersion: "25890" uid: 64cc08ad-d6fd-4363-9d99-bf4af9fa3230 kind: Event lastTimestamp: "2026-07-20T10:30:41Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:30:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:30:41Z" name: success-200-isvc-a956e-predictor.18c3f8caa9b7d119 namespace: kserve-ci-e2e-test resourceVersion: "26237" uid: 9c7d6f58-184e-471e-be28-00bc0874d04d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-20T10:30:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a956e-predictor namespace: kserve-ci-e2e-test resourceVersion: "25890" uid: 64cc08ad-d6fd-4363-9d99-bf4af9fa3230 kind: Event lastTimestamp: "2026-07-20T10:30:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:30:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:30:41Z" name: success-200-isvc-a956e-predictor.18c3f8caaa02a716 namespace: kserve-ci-e2e-test resourceVersion: "26238" uid: 4acc1c87-d8a3-48c5-af33-38664c4e1c33 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-20T10:30:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a956e-predictor namespace: kserve-ci-e2e-test resourceVersion: "26126" uid: 64cc08ad-d6fd-4363-9d99-bf4af9fa3230 kind: Event lastTimestamp: "2026-07-20T10:31:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:30:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:31:11Z" name: success-200-isvc-a956e-predictor.18c3f8d1a8024ae5 namespace: kserve-ci-e2e-test resourceVersion: "26599" uid: e04ea016-4798-459f-9211-8b80dbdfefc2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-20T10:30:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a956e-predictor namespace: kserve-ci-e2e-test resourceVersion: "26126" uid: 64cc08ad-d6fd-4363-9d99-bf4af9fa3230 kind: Event lastTimestamp: "2026-07-20T10:31:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:30:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:31:11Z" name: success-200-isvc-a956e-predictor.18c3f8d1a875240b namespace: kserve-ci-e2e-test resourceVersion: "26602" uid: bb5f1754-a0e9-496e-acc3-0b9e8058c092 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-20T10:30:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a956e namespace: kserve-ci-e2e-test resourceVersion: "25880" uid: 8bb491b6-6f38-4733-aed2-40b88a3f351a kind: Event lastTimestamp: "2026-07-20T10:30:59Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-20T10:30:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:30:59Z" name: success-200-isvc-a956e.18c3f8c72bcf4bff namespace: kserve-ci-e2e-test resourceVersion: "26397" uid: ebd4298b-567e-425e-a363-bdd7c32854a5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:30:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a956e namespace: kserve-ci-e2e-test resourceVersion: "25880" uid: 8bb491b6-6f38-4733-aed2-40b88a3f351a kind: Event lastTimestamp: "2026-07-20T10:30:11Z" message: 'Failed to update status for InferenceService "success-200-isvc-a956e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-a956e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-20T10:30:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:30:11Z" name: success-200-isvc-a956e.18c3f8c72c36d3e7 namespace: kserve-ci-e2e-test resourceVersion: "25896" uid: a3493f5c-59a7-48e4-9fad-5900eb9e6e8f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:30:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a956e namespace: kserve-ci-e2e-test resourceVersion: "25880" uid: 8bb491b6-6f38-4733-aed2-40b88a3f351a kind: Event lastTimestamp: "2026-07-20T10:30:11Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-a956e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-20T10:30:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:30:11Z" name: success-200-isvc-a956e.18c3f8c72c3731f6 namespace: kserve-ci-e2e-test resourceVersion: "25897" uid: 6a9b66cf-c229-495e-8b91-280038338bb1 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:30:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a956e namespace: kserve-ci-e2e-test resourceVersion: "26393" uid: 8bb491b6-6f38-4733-aed2-40b88a3f351a kind: Event lastTimestamp: "2026-07-20T10:30:59Z" message: InferenceService [success-200-isvc-a956e] is Ready metadata: creationTimestamp: "2026-07-20T10:30:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:30:59Z" name: success-200-isvc-a956e.18c3f8d2550bf373 namespace: kserve-ci-e2e-test resourceVersion: "26396" uid: 232d275c-e879-40d7-a788-c48c3a12de5d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-20T10:09:01.386028Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ab339-predictor-559b85c779-xld8k namespace: kserve-ci-e2e-test resourceVersion: "14891" uid: 23772513-4c41-42bb-aa78-c966ae1f0ad5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-ab339-predictor-559b85c779-xld8k to ip-10-0-143-155.ec2.internal metadata: creationTimestamp: "2026-07-20T10:09:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-20T10:09:01Z" name: success-200-isvc-ab339-predictor-559b85c779-xld8k.18c3f79f680791a8 namespace: kserve-ci-e2e-test resourceVersion: "14897" uid: 499aa9df-b314-4919-bcf9-784b0c5ce65f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5dc669d9-69hkz source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:01Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ab339-predictor-559b85c779-xld8k namespace: kserve-ci-e2e-test resourceVersion: "14895" uid: 23772513-4c41-42bb-aa78-c966ae1f0ad5 kind: Event lastTimestamp: "2026-07-20T10:09:01Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-ab339-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-20T10:09:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:09:01Z" name: success-200-isvc-ab339-predictor-559b85c779-xld8k.18c3f79f771f1387 namespace: kserve-ci-e2e-test resourceVersion: "14954" uid: fcfda248-16d1-4216-a9c0-ec38c5c838e5 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:02Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ab339-predictor-559b85c779-xld8k namespace: kserve-ci-e2e-test resourceVersion: "14901" uid: 23772513-4c41-42bb-aa78-c966ae1f0ad5 kind: Event lastTimestamp: "2026-07-20T10:09:02Z" message: Add eth0 [10.133.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-20T10:09:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-20T10:09:02Z" name: success-200-isvc-ab339-predictor-559b85c779-xld8k.18c3f79fa58ae56e namespace: kserve-ci-e2e-test resourceVersion: "15017" uid: 1603102d-723b-4c81-8654-09a1aa358ee3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ab339-predictor-559b85c779-xld8k namespace: kserve-ci-e2e-test resourceVersion: "14895" uid: 23772513-4c41-42bb-aa78-c966ae1f0ad5 kind: Event lastTimestamp: "2026-07-20T10:09:02Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1771" metadata: creationTimestamp: "2026-07-20T10:09:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:09:02Z" name: success-200-isvc-ab339-predictor-559b85c779-xld8k.18c3f79fa7374a26 namespace: kserve-ci-e2e-test resourceVersion: "15019" uid: 321c8f63-a79f-478e-bf91-005647939ce5 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ab339-predictor-559b85c779-xld8k namespace: kserve-ci-e2e-test resourceVersion: "14895" uid: 23772513-4c41-42bb-aa78-c966ae1f0ad5 kind: Event lastTimestamp: "2026-07-20T10:09:19Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1771" in 16.969s (16.969s including waiting). Image size: 1335516782 bytes.' metadata: creationTimestamp: "2026-07-20T10:09:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:09:19Z" name: success-200-isvc-ab339-predictor-559b85c779-xld8k.18c3f7a39aae07e8 namespace: kserve-ci-e2e-test resourceVersion: "15179" uid: 595968e5-9662-4c7e-8a30-7d9bbc5a634d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ab339-predictor-559b85c779-xld8k namespace: kserve-ci-e2e-test resourceVersion: "14895" uid: 23772513-4c41-42bb-aa78-c966ae1f0ad5 kind: Event lastTimestamp: "2026-07-20T10:09:19Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-20T10:09:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:09:19Z" name: success-200-isvc-ab339-predictor-559b85c779-xld8k.18c3f7a39f031bd6 namespace: kserve-ci-e2e-test resourceVersion: "15184" uid: 8ae98862-c598-4b5f-9dc8-f6183d1e0ae7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ab339-predictor-559b85c779-xld8k namespace: kserve-ci-e2e-test resourceVersion: "14895" uid: 23772513-4c41-42bb-aa78-c966ae1f0ad5 kind: Event lastTimestamp: "2026-07-20T10:09:19Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-20T10:09:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:09:19Z" name: success-200-isvc-ab339-predictor-559b85c779-xld8k.18c3f7a39f942359 namespace: kserve-ci-e2e-test resourceVersion: "15187" uid: 401a72d2-a2dd-408f-822a-e5941c386f15 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ab339-predictor-559b85c779-xld8k namespace: kserve-ci-e2e-test resourceVersion: "14895" uid: 23772513-4c41-42bb-aa78-c966ae1f0ad5 kind: Event lastTimestamp: "2026-07-20T10:09:19Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-20T10:09:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:09:19Z" name: success-200-isvc-ab339-predictor-559b85c779-xld8k.18c3f7a39fa1bc83 namespace: kserve-ci-e2e-test resourceVersion: "15188" uid: 0606d180-6553-470f-8ec9-33ccc8ab7bac reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ab339-predictor-559b85c779-xld8k namespace: kserve-ci-e2e-test resourceVersion: "14895" uid: 23772513-4c41-42bb-aa78-c966ae1f0ad5 kind: Event lastTimestamp: "2026-07-20T10:09:22Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.517s (2.517s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-20T10:09:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:09:22Z" name: success-200-isvc-ab339-predictor-559b85c779-xld8k.18c3f7a435b48f6d namespace: kserve-ci-e2e-test resourceVersion: "15212" uid: bb6072c8-287a-4543-bdd0-285217490ab0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ab339-predictor-559b85c779-xld8k namespace: kserve-ci-e2e-test resourceVersion: "14895" uid: 23772513-4c41-42bb-aa78-c966ae1f0ad5 kind: Event lastTimestamp: "2026-07-20T10:09:22Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-20T10:09:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:09:22Z" name: success-200-isvc-ab339-predictor-559b85c779-xld8k.18c3f7a43901c83a namespace: kserve-ci-e2e-test resourceVersion: "15214" uid: cdec864a-a79e-4c13-a79d-b331355d70a8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ab339-predictor-559b85c779-xld8k namespace: kserve-ci-e2e-test resourceVersion: "14895" uid: 23772513-4c41-42bb-aa78-c966ae1f0ad5 kind: Event lastTimestamp: "2026-07-20T10:09:22Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:09:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:09:22Z" name: success-200-isvc-ab339-predictor-559b85c779-xld8k.18c3f7a4397289f2 namespace: kserve-ci-e2e-test resourceVersion: "15216" uid: b3929506-23d0-4e89-aa2c-c10e2b048f92 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-20T10:09:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ab339-predictor-559b85c779-xld8k namespace: kserve-ci-e2e-test resourceVersion: "14895" uid: 23772513-4c41-42bb-aa78-c966ae1f0ad5 kind: Event lastTimestamp: "2026-07-20T10:09:58Z" message: 'Readiness probe failed: dial tcp 10.133.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-20T10:09:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:09:58Z" name: success-200-isvc-ab339-predictor-559b85c779-xld8k.18c3f7a44cf28bbf namespace: kserve-ci-e2e-test resourceVersion: "15547" uid: 7afbad08-bf24-40f1-995b-d48f9faaaf26 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ab339-predictor-559b85c779-xld8k namespace: kserve-ci-e2e-test resourceVersion: "14895" uid: 23772513-4c41-42bb-aa78-c966ae1f0ad5 kind: Event lastTimestamp: "2026-07-20T10:10:31Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-20T10:10:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:10:31Z" name: success-200-isvc-ab339-predictor-559b85c779-xld8k.18c3f7b457e9d9f6 namespace: kserve-ci-e2e-test resourceVersion: "15907" uid: 0c8cb95b-c24e-44c9-9dce-9fe9fa32a612 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ab339-predictor-559b85c779-xld8k namespace: kserve-ci-e2e-test resourceVersion: "14895" uid: 23772513-4c41-42bb-aa78-c966ae1f0ad5 kind: Event lastTimestamp: "2026-07-20T10:10:31Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-20T10:10:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:10:31Z" name: success-200-isvc-ab339-predictor-559b85c779-xld8k.18c3f7b457ea8c55 namespace: kserve-ci-e2e-test resourceVersion: "15908" uid: 88bba6b9-d56f-4c3f-8853-55121425d601 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ab339-predictor-559b85c779-xld8k namespace: kserve-ci-e2e-test resourceVersion: "14895" uid: 23772513-4c41-42bb-aa78-c966ae1f0ad5 kind: Event lastTimestamp: "2026-07-20T10:10:33Z" message: 'Readiness probe failed: Get "https://10.133.0.36:8643/healthz": dial tcp 10.133.0.36:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-20T10:10:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:10:33Z" name: success-200-isvc-ab339-predictor-559b85c779-xld8k.18c3f7b4d560f169 namespace: kserve-ci-e2e-test resourceVersion: "16031" uid: 37489ec6-d85e-42bf-b079-f06a435b932e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-155.ec2.internal source: component: kubelet host: ip-10-0-143-155.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-ab339-predictor-559b85c779 namespace: kserve-ci-e2e-test resourceVersion: "14884" uid: 8aa10d6b-a253-46e0-bfec-dd69c25f5cce kind: Event lastTimestamp: "2026-07-20T10:09:01Z" message: 'Created pod: success-200-isvc-ab339-predictor-559b85c779-xld8k' metadata: creationTimestamp: "2026-07-20T10:09:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:09:01Z" name: success-200-isvc-ab339-predictor-559b85c779.18c3f79f678e83a8 namespace: kserve-ci-e2e-test resourceVersion: "14894" uid: 5f465481-8905-4dcb-8516-7433884c5b54 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:09:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-ab339-predictor namespace: kserve-ci-e2e-test resourceVersion: "14881" uid: 5c0a287d-343d-44f1-a453-41543de6f791 kind: Event lastTimestamp: "2026-07-20T10:09:01Z" message: Scaled up replica set success-200-isvc-ab339-predictor-559b85c779 from 0 to 1 metadata: creationTimestamp: "2026-07-20T10:09:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:09:01Z" name: success-200-isvc-ab339-predictor.18c3f79f64c90cda namespace: kserve-ci-e2e-test resourceVersion: "14887" uid: 124f341c-5db6-4dd5-b118-50aa694540bf reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-20T10:09:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ab339-predictor namespace: kserve-ci-e2e-test resourceVersion: "14883" uid: d2fbc2a0-8058-4cd2-ac22-c94293d96214 kind: Event lastTimestamp: "2026-07-20T10:09:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:09:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:09:46Z" name: success-200-isvc-ab339-predictor.18c3f7a2e30fe2c8 namespace: kserve-ci-e2e-test resourceVersion: "15449" uid: 18234d76-d495-4027-b86d-2a9f3e255444 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-20T10:09:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ab339-predictor namespace: kserve-ci-e2e-test resourceVersion: "14883" uid: d2fbc2a0-8058-4cd2-ac22-c94293d96214 kind: Event lastTimestamp: "2026-07-20T10:09:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:09:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:09:46Z" name: success-200-isvc-ab339-predictor.18c3f7a2e3733ba1 namespace: kserve-ci-e2e-test resourceVersion: "15450" uid: dc4e0b27-feda-4305-95ca-f766af0febf9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-20T10:10:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ab339-predictor namespace: kserve-ci-e2e-test resourceVersion: "15147" uid: d2fbc2a0-8058-4cd2-ac22-c94293d96214 kind: Event lastTimestamp: "2026-07-20T10:10:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:10:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:10:16Z" name: success-200-isvc-ab339-predictor.18c3f7ad5fa2abd2 namespace: kserve-ci-e2e-test resourceVersion: "15701" uid: 35388aaa-fcd8-44bc-9c75-d0f0cf135f2e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-20T10:10:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ab339-predictor namespace: kserve-ci-e2e-test resourceVersion: "15147" uid: d2fbc2a0-8058-4cd2-ac22-c94293d96214 kind: Event lastTimestamp: "2026-07-20T10:10:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:10:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:10:16Z" name: success-200-isvc-ab339-predictor.18c3f7ad60059ccc namespace: kserve-ci-e2e-test resourceVersion: "15702" uid: 7d539f5b-4ce0-4b5d-a68a-9bf0d4b4c98c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-20T10:09:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ab339 namespace: kserve-ci-e2e-test resourceVersion: "14872" uid: 7d02cb52-8535-4cfb-8207-63d5f8cee853 kind: Event lastTimestamp: "2026-07-20T10:10:08Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-20T10:09:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:10:08Z" name: success-200-isvc-ab339.18c3f79f64cbecc1 namespace: kserve-ci-e2e-test resourceVersion: "15647" uid: 4962d93d-bedb-49d9-92dc-2aaa7dac88eb reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ab339 namespace: kserve-ci-e2e-test resourceVersion: "15644" uid: 7d02cb52-8535-4cfb-8207-63d5f8cee853 kind: Event lastTimestamp: "2026-07-20T10:10:08Z" message: InferenceService [success-200-isvc-ab339] is Ready metadata: creationTimestamp: "2026-07-20T10:10:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:10:08Z" name: success-200-isvc-ab339.18c3f7af0bc0566b namespace: kserve-ci-e2e-test resourceVersion: "15645" uid: 49e57ec0-8e56-4163-91e9-9ff23543017d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-20T10:32:11.530739Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-78ab0-55d644b55b-t2xws namespace: kserve-ci-e2e-test resourceVersion: "27349" uid: f393f1ee-7ac2-428a-a1b4-d3eb85223f02 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-78ab0-55d644b55b-t2xws to ip-10-0-138-143.ec2.internal metadata: creationTimestamp: "2026-07-20T10:32:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-20T10:32:11Z" name: switch-graph-78ab0-55d644b55b-t2xws.18c3f8e3131c715a namespace: kserve-ci-e2e-test resourceVersion: "27355" uid: 94c526ce-489c-414e-a55e-386531c4ae86 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5dc669d9-69hkz source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:32:11Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-78ab0-55d644b55b-t2xws namespace: kserve-ci-e2e-test resourceVersion: "27352" uid: f393f1ee-7ac2-428a-a1b4-d3eb85223f02 kind: Event lastTimestamp: "2026-07-20T10:32:11Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-78ab0-serving-cert" not found' metadata: creationTimestamp: "2026-07-20T10:32:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:32:11Z" name: switch-graph-78ab0-55d644b55b-t2xws.18c3f8e320186609 namespace: kserve-ci-e2e-test resourceVersion: "27366" uid: 0056d013-9f08-4a3b-b703-21c0cf96535f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:32:12Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-78ab0-55d644b55b-t2xws namespace: kserve-ci-e2e-test resourceVersion: "27358" uid: f393f1ee-7ac2-428a-a1b4-d3eb85223f02 kind: Event lastTimestamp: "2026-07-20T10:32:12Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-20T10:32:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-20T10:32:12Z" name: switch-graph-78ab0-55d644b55b-t2xws.18c3f8e35019e726 namespace: kserve-ci-e2e-test resourceVersion: "27373" uid: 8d1ca495-a1be-44ad-acd9-982b3b443015 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:32:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-78ab0} kind: Pod name: switch-graph-78ab0-55d644b55b-t2xws namespace: kserve-ci-e2e-test resourceVersion: "27352" uid: f393f1ee-7ac2-428a-a1b4-d3eb85223f02 kind: Event lastTimestamp: "2026-07-20T10:32:12Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:ee73d7a22df8b79aa8a5e917cb435b1d2b100d4582439c437bdc01a1d2422290" already present on machine metadata: creationTimestamp: "2026-07-20T10:32:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:32:12Z" name: switch-graph-78ab0-55d644b55b-t2xws.18c3f8e351820484 namespace: kserve-ci-e2e-test resourceVersion: "27375" uid: 32739400-10cb-46ab-8ed4-a324c0f6c37a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:32:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-78ab0} kind: Pod name: switch-graph-78ab0-55d644b55b-t2xws namespace: kserve-ci-e2e-test resourceVersion: "27352" uid: f393f1ee-7ac2-428a-a1b4-d3eb85223f02 kind: Event lastTimestamp: "2026-07-20T10:32:12Z" message: 'Created container: switch-graph-78ab0' metadata: creationTimestamp: "2026-07-20T10:32:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:32:12Z" name: switch-graph-78ab0-55d644b55b-t2xws.18c3f8e3559c7f4e namespace: kserve-ci-e2e-test resourceVersion: "27376" uid: a012b5af-cfd4-4d88-9811-4c193ee9910a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:32:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-78ab0} kind: Pod name: switch-graph-78ab0-55d644b55b-t2xws namespace: kserve-ci-e2e-test resourceVersion: "27352" uid: f393f1ee-7ac2-428a-a1b4-d3eb85223f02 kind: Event lastTimestamp: "2026-07-20T10:32:12Z" message: Started container switch-graph-78ab0 metadata: creationTimestamp: "2026-07-20T10:32:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:32:12Z" name: switch-graph-78ab0-55d644b55b-t2xws.18c3f8e3560950a7 namespace: kserve-ci-e2e-test resourceVersion: "27377" uid: 9788a903-2879-40cc-a4c1-c9f7caa8ff94 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:48:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-78ab0} kind: Pod name: switch-graph-78ab0-55d644b55b-t2xws namespace: kserve-ci-e2e-test resourceVersion: "27352" uid: f393f1ee-7ac2-428a-a1b4-d3eb85223f02 kind: Event lastTimestamp: "2026-07-20T10:48:30Z" message: Stopping container switch-graph-78ab0 metadata: creationTimestamp: "2026-07-20T10:48:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:48:30Z" name: switch-graph-78ab0-55d644b55b-t2xws.18c3f9c704868067 namespace: kserve-ci-e2e-test resourceVersion: "34251" uid: e5c1fdd6-e58e-4c79-a7e4-a794f5613769 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-20T10:48:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-78ab0} kind: Pod name: switch-graph-78ab0-55d644b55b-t2xws namespace: kserve-ci-e2e-test resourceVersion: "27352" uid: f393f1ee-7ac2-428a-a1b4-d3eb85223f02 kind: Event lastTimestamp: "2026-07-20T10:48:54Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-20T10:48:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:48:54Z" name: switch-graph-78ab0-55d644b55b-t2xws.18c3f9c7f81eec07 namespace: kserve-ci-e2e-test resourceVersion: "34496" uid: 4f16eccc-fde5-4262-aa40-aecf95352d37 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:32:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-78ab0-55d644b55b namespace: kserve-ci-e2e-test resourceVersion: "27345" uid: 8318139a-f913-4895-83f6-f2d46016365b kind: Event lastTimestamp: "2026-07-20T10:32:11Z" message: 'Created pod: switch-graph-78ab0-55d644b55b-t2xws' metadata: creationTimestamp: "2026-07-20T10:32:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:32:11Z" name: switch-graph-78ab0-55d644b55b.18c3f8e3121dfcbf namespace: kserve-ci-e2e-test resourceVersion: "27351" uid: bc5d6615-8995-43de-9eb0-aae3abbf511b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:32:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-78ab0 namespace: kserve-ci-e2e-test resourceVersion: "27342" uid: b3c29701-0190-45f4-87a7-baa86d294959 kind: Event lastTimestamp: "2026-07-20T10:32:11Z" message: Scaled up replica set switch-graph-78ab0-55d644b55b from 0 to 1 metadata: creationTimestamp: "2026-07-20T10:32:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:32:11Z" name: switch-graph-78ab0.18c3f8e3105c9cf2 namespace: kserve-ci-e2e-test resourceVersion: "27347" uid: 4188a0ce-8883-486a-85ef-abb4b5e9860c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:32:11Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-78ab0 namespace: kserve-ci-e2e-test resourceVersion: "27360" uid: 123a15c6-e640-4b58-b8f8-af0c4aa6322a kind: Event lastTimestamp: "2026-07-20T10:32:11Z" message: InferenceGraph [switch-graph-78ab0] is Ready metadata: creationTimestamp: "2026-07-20T10:32:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:32:11Z" name: switch-graph-78ab0.18c3f8e31818718a namespace: kserve-ci-e2e-test resourceVersion: "27363" uid: 74e063bb-3b09-4f1c-926f-5c214d895377 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:32:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-78ab0 namespace: kserve-ci-e2e-test resourceVersion: "27343" uid: 42cb9be9-cf3f-4308-a313-8b8b31cfcdb4 kind: Event lastTimestamp: "2026-07-20T10:32:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:32:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:32:26Z" name: switch-graph-78ab0.18c3f8e68eefd18e namespace: kserve-ci-e2e-test resourceVersion: "27496" uid: 935eab74-ed80-4172-a432-91ce6f1f3d50 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:32:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-78ab0 namespace: kserve-ci-e2e-test resourceVersion: "27343" uid: 42cb9be9-cf3f-4308-a313-8b8b31cfcdb4 kind: Event lastTimestamp: "2026-07-20T10:32:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-20T10:32:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:32:26Z" name: switch-graph-78ab0.18c3f8e68f38a50c namespace: kserve-ci-e2e-test resourceVersion: "27498" uid: ec7537c0-3ab6-4cf1-9101-dadacf227f54 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:32:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-78ab0 namespace: kserve-ci-e2e-test resourceVersion: "27497" uid: 42cb9be9-cf3f-4308-a313-8b8b31cfcdb4 kind: Event lastTimestamp: "2026-07-20T10:32:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:32:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:32:41Z" name: switch-graph-78ab0.18c3f8ea0de07183 namespace: kserve-ci-e2e-test resourceVersion: "27649" uid: 8f832cd9-24bf-462d-b25e-30e808605947 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:32:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-78ab0 namespace: kserve-ci-e2e-test resourceVersion: "27497" uid: 42cb9be9-cf3f-4308-a313-8b8b31cfcdb4 kind: Event lastTimestamp: "2026-07-20T10:32:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:32:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:32:41Z" name: switch-graph-78ab0.18c3f8ea0e47533a namespace: kserve-ci-e2e-test resourceVersion: "27651" uid: dd61b354-a8ab-41f9-a12b-ced515c0b159 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-20T10:11:31.524276Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-7f819-56ff4d7f75-tqt8m namespace: kserve-ci-e2e-test resourceVersion: "16755" uid: 5a7862ab-fc4c-460a-bbaf-e81634db9df3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-7f819-56ff4d7f75-tqt8m to ip-10-0-138-143.ec2.internal metadata: creationTimestamp: "2026-07-20T10:11:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-20T10:11:31Z" name: switch-graph-7f819-56ff4d7f75-tqt8m.18c3f7c25cf784a1 namespace: kserve-ci-e2e-test resourceVersion: "16760" uid: b1d56e34-ce38-45b3-a253-0750933e006d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5dc669d9-69hkz source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:31Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-7f819-56ff4d7f75-tqt8m namespace: kserve-ci-e2e-test resourceVersion: "16758" uid: 5a7862ab-fc4c-460a-bbaf-e81634db9df3 kind: Event lastTimestamp: "2026-07-20T10:11:31Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-7f819-serving-cert" not found' metadata: creationTimestamp: "2026-07-20T10:11:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:11:31Z" name: switch-graph-7f819-56ff4d7f75-tqt8m.18c3f7c2656f1838 namespace: kserve-ci-e2e-test resourceVersion: "16774" uid: 49ab22dd-1485-49d1-ae99-2ebf1eeda567 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:32Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-7f819-56ff4d7f75-tqt8m namespace: kserve-ci-e2e-test resourceVersion: "16765" uid: 5a7862ab-fc4c-460a-bbaf-e81634db9df3 kind: Event lastTimestamp: "2026-07-20T10:11:32Z" message: Add eth0 [10.134.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-20T10:11:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-20T10:11:32Z" name: switch-graph-7f819-56ff4d7f75-tqt8m.18c3f7c29a27b3dc namespace: kserve-ci-e2e-test resourceVersion: "16781" uid: aff55065-f6cb-4075-ac1d-f00c2ecf4f2e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-7f819} kind: Pod name: switch-graph-7f819-56ff4d7f75-tqt8m namespace: kserve-ci-e2e-test resourceVersion: "16758" uid: 5a7862ab-fc4c-460a-bbaf-e81634db9df3 kind: Event lastTimestamp: "2026-07-20T10:11:32Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:ee73d7a22df8b79aa8a5e917cb435b1d2b100d4582439c437bdc01a1d2422290" already present on machine metadata: creationTimestamp: "2026-07-20T10:11:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:11:32Z" name: switch-graph-7f819-56ff4d7f75-tqt8m.18c3f7c29bbc72ad namespace: kserve-ci-e2e-test resourceVersion: "16783" uid: 659fc245-19f3-4967-9410-060e902bdddc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-7f819} kind: Pod name: switch-graph-7f819-56ff4d7f75-tqt8m namespace: kserve-ci-e2e-test resourceVersion: "16758" uid: 5a7862ab-fc4c-460a-bbaf-e81634db9df3 kind: Event lastTimestamp: "2026-07-20T10:11:32Z" message: 'Created container: switch-graph-7f819' metadata: creationTimestamp: "2026-07-20T10:11:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:11:32Z" name: switch-graph-7f819-56ff4d7f75-tqt8m.18c3f7c29f0a8e13 namespace: kserve-ci-e2e-test resourceVersion: "16784" uid: 2ac1c4ed-7606-438c-b283-6fddff20b669 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-7f819} kind: Pod name: switch-graph-7f819-56ff4d7f75-tqt8m namespace: kserve-ci-e2e-test resourceVersion: "16758" uid: 5a7862ab-fc4c-460a-bbaf-e81634db9df3 kind: Event lastTimestamp: "2026-07-20T10:11:32Z" message: Started container switch-graph-7f819 metadata: creationTimestamp: "2026-07-20T10:11:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:11:32Z" name: switch-graph-7f819-56ff4d7f75-tqt8m.18c3f7c29f7bfd76 namespace: kserve-ci-e2e-test resourceVersion: "16785" uid: b8ef83d2-8665-4e2a-884f-2182d2198511 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:19:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-7f819} kind: Pod name: switch-graph-7f819-56ff4d7f75-tqt8m namespace: kserve-ci-e2e-test resourceVersion: "16758" uid: 5a7862ab-fc4c-460a-bbaf-e81634db9df3 kind: Event lastTimestamp: "2026-07-20T10:19:46Z" message: Stopping container switch-graph-7f819 metadata: creationTimestamp: "2026-07-20T10:19:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:19:46Z" name: switch-graph-7f819-56ff4d7f75-tqt8m.18c3f8358f8b57ab namespace: kserve-ci-e2e-test resourceVersion: "20296" uid: df3339f3-fb4d-494e-8ee6-933b381f2968 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-20T10:19:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-7f819} kind: Pod name: switch-graph-7f819-56ff4d7f75-tqt8m namespace: kserve-ci-e2e-test resourceVersion: "16758" uid: 5a7862ab-fc4c-460a-bbaf-e81634db9df3 kind: Event lastTimestamp: "2026-07-20T10:20:13Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-20T10:19:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:20:13Z" name: switch-graph-7f819-56ff4d7f75-tqt8m.18c3f8362ca14515 namespace: kserve-ci-e2e-test resourceVersion: "20632" uid: 073a3909-ca86-4102-95d3-227d6f76fb87 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-7f819-56ff4d7f75 namespace: kserve-ci-e2e-test resourceVersion: "16752" uid: 3ea69c03-8e5a-4884-9113-ce829b46070d kind: Event lastTimestamp: "2026-07-20T10:11:31Z" message: 'Created pod: switch-graph-7f819-56ff4d7f75-tqt8m' metadata: creationTimestamp: "2026-07-20T10:11:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:11:31Z" name: switch-graph-7f819-56ff4d7f75.18c3f7c25c83a226 namespace: kserve-ci-e2e-test resourceVersion: "16756" uid: 6974c983-d17e-416e-8fd7-02941cb2e4a9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-7f819 namespace: kserve-ci-e2e-test resourceVersion: "16750" uid: b7b09bd1-7d3b-4bee-9f93-f678d0ae602d kind: Event lastTimestamp: "2026-07-20T10:11:31Z" message: Scaled up replica set switch-graph-7f819-56ff4d7f75 from 0 to 1 metadata: creationTimestamp: "2026-07-20T10:11:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:11:31Z" name: switch-graph-7f819.18c3f7c25bd56e0d namespace: kserve-ci-e2e-test resourceVersion: "16753" uid: 410d4718-b39a-4ff1-9c54-967ecc176b62 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:31Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-7f819 namespace: kserve-ci-e2e-test resourceVersion: "16770" uid: 5ff7052d-3025-4e79-9865-30a087a44cd9 kind: Event lastTimestamp: "2026-07-20T10:11:31Z" message: InferenceGraph [switch-graph-7f819] is Ready metadata: creationTimestamp: "2026-07-20T10:11:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:11:31Z" name: switch-graph-7f819.18c3f7c26158404d namespace: kserve-ci-e2e-test resourceVersion: "16772" uid: bd666d3c-1945-4f97-9f0e-3a0c6c81b637 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-7f819 namespace: kserve-ci-e2e-test resourceVersion: "16751" uid: 3f2098ba-7588-4098-9d26-668a6c3a2197 kind: Event lastTimestamp: "2026-07-20T10:11:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:11:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:11:46Z" name: switch-graph-7f819.18c3f7c5dad9e238 namespace: kserve-ci-e2e-test resourceVersion: "16911" uid: 422d0cf8-1005-4181-a889-2871ae1ec76f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:11:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-7f819 namespace: kserve-ci-e2e-test resourceVersion: "16751" uid: 3f2098ba-7588-4098-9d26-668a6c3a2197 kind: Event lastTimestamp: "2026-07-20T10:11:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-20T10:11:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:11:46Z" name: switch-graph-7f819.18c3f7c5dbdedf2d namespace: kserve-ci-e2e-test resourceVersion: "16913" uid: b3393869-84d0-44ce-b885-39f16727f31f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-20T10:10:21.085523Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-ab339-7df5ddcb69-75wdq namespace: kserve-ci-e2e-test resourceVersion: "15756" uid: d9427823-77a6-4838-96d5-e14af568d7d7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-ab339-7df5ddcb69-75wdq to ip-10-0-138-143.ec2.internal metadata: creationTimestamp: "2026-07-20T10:10:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-20T10:10:21Z" name: switch-graph-ab339-7df5ddcb69-75wdq.18c3f7b1f67d55ce namespace: kserve-ci-e2e-test resourceVersion: "15761" uid: a66f2a8e-91ca-4164-84dc-2533a0b41f2f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c5dc669d9-69hkz source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:21Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-ab339-7df5ddcb69-75wdq namespace: kserve-ci-e2e-test resourceVersion: "15759" uid: d9427823-77a6-4838-96d5-e14af568d7d7 kind: Event lastTimestamp: "2026-07-20T10:10:21Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-ab339-serving-cert" not found' metadata: creationTimestamp: "2026-07-20T10:10:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:10:21Z" name: switch-graph-ab339-7df5ddcb69-75wdq.18c3f7b1feb338e3 namespace: kserve-ci-e2e-test resourceVersion: "15774" uid: 3df6794d-f5f4-4e7e-b2ff-f414c2090dd3 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:22Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-ab339-7df5ddcb69-75wdq namespace: kserve-ci-e2e-test resourceVersion: "15765" uid: d9427823-77a6-4838-96d5-e14af568d7d7 kind: Event lastTimestamp: "2026-07-20T10:10:22Z" message: Add eth0 [10.134.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-20T10:10:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-20T10:10:22Z" name: switch-graph-ab339-7df5ddcb69-75wdq.18c3f7b233b50308 namespace: kserve-ci-e2e-test resourceVersion: "15784" uid: e80b4fd9-ae9a-4641-bb7e-be96c85da7f2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-ab339} kind: Pod name: switch-graph-ab339-7df5ddcb69-75wdq namespace: kserve-ci-e2e-test resourceVersion: "15759" uid: d9427823-77a6-4838-96d5-e14af568d7d7 kind: Event lastTimestamp: "2026-07-20T10:10:22Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:ee73d7a22df8b79aa8a5e917cb435b1d2b100d4582439c437bdc01a1d2422290" metadata: creationTimestamp: "2026-07-20T10:10:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:10:22Z" name: switch-graph-ab339-7df5ddcb69-75wdq.18c3f7b235085e19 namespace: kserve-ci-e2e-test resourceVersion: "15786" uid: 6f601ceb-dc4e-4e55-9a74-4f311860167f reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-ab339} kind: Pod name: switch-graph-ab339-7df5ddcb69-75wdq namespace: kserve-ci-e2e-test resourceVersion: "15759" uid: d9427823-77a6-4838-96d5-e14af568d7d7 kind: Event lastTimestamp: "2026-07-20T10:10:24Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:ee73d7a22df8b79aa8a5e917cb435b1d2b100d4582439c437bdc01a1d2422290" in 2.118s (2.118s including waiting). Image size: 218228855 bytes.' metadata: creationTimestamp: "2026-07-20T10:10:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:10:24Z" name: switch-graph-ab339-7df5ddcb69-75wdq.18c3f7b2b3525204 namespace: kserve-ci-e2e-test resourceVersion: "15803" uid: 6fe138be-0bff-4428-8fda-facd849271f4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-ab339} kind: Pod name: switch-graph-ab339-7df5ddcb69-75wdq namespace: kserve-ci-e2e-test resourceVersion: "15759" uid: d9427823-77a6-4838-96d5-e14af568d7d7 kind: Event lastTimestamp: "2026-07-20T10:10:24Z" message: 'Created container: switch-graph-ab339' metadata: creationTimestamp: "2026-07-20T10:10:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:10:24Z" name: switch-graph-ab339-7df5ddcb69-75wdq.18c3f7b2b6443224 namespace: kserve-ci-e2e-test resourceVersion: "15804" uid: e254be02-1ba1-4f7b-8f53-9521b12e3991 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-ab339} kind: Pod name: switch-graph-ab339-7df5ddcb69-75wdq namespace: kserve-ci-e2e-test resourceVersion: "15759" uid: d9427823-77a6-4838-96d5-e14af568d7d7 kind: Event lastTimestamp: "2026-07-20T10:10:24Z" message: Started container switch-graph-ab339 metadata: creationTimestamp: "2026-07-20T10:10:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:10:24Z" name: switch-graph-ab339-7df5ddcb69-75wdq.18c3f7b2b6ae5d9d namespace: kserve-ci-e2e-test resourceVersion: "15805" uid: bf44ca4a-676a-4b40-9b27-3f5d8fe4722d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-ab339} kind: Pod name: switch-graph-ab339-7df5ddcb69-75wdq namespace: kserve-ci-e2e-test resourceVersion: "15759" uid: d9427823-77a6-4838-96d5-e14af568d7d7 kind: Event lastTimestamp: "2026-07-20T10:10:31Z" message: Stopping container switch-graph-ab339 metadata: creationTimestamp: "2026-07-20T10:10:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:10:31Z" name: switch-graph-ab339-7df5ddcb69-75wdq.18c3f7b451e87c48 namespace: kserve-ci-e2e-test resourceVersion: "15883" uid: 0bf64763-6c2f-459f-856e-73d810bdc166 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-20T10:10:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-ab339} kind: Pod name: switch-graph-ab339-7df5ddcb69-75wdq namespace: kserve-ci-e2e-test resourceVersion: "15759" uid: d9427823-77a6-4838-96d5-e14af568d7d7 kind: Event lastTimestamp: "2026-07-20T10:11:00Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-20T10:10:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-20T10:11:00Z" name: switch-graph-ab339-7df5ddcb69-75wdq.18c3f7b55c11a0db namespace: kserve-ci-e2e-test resourceVersion: "16270" uid: 0b6f6e56-0538-41ac-8760-04395b116fd8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-143.ec2.internal source: component: kubelet host: ip-10-0-138-143.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-ab339-7df5ddcb69 namespace: kserve-ci-e2e-test resourceVersion: "15753" uid: 84fb7f24-b9fc-4119-a8e8-d0a0f5073679 kind: Event lastTimestamp: "2026-07-20T10:10:21Z" message: 'Created pod: switch-graph-ab339-7df5ddcb69-75wdq' metadata: creationTimestamp: "2026-07-20T10:10:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:10:21Z" name: switch-graph-ab339-7df5ddcb69.18c3f7b1f6173dd1 namespace: kserve-ci-e2e-test resourceVersion: "15758" uid: 173e56e0-297a-4821-9b35-794d14d27d2a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-ab339 namespace: kserve-ci-e2e-test resourceVersion: "15751" uid: 2ef4c014-7b50-410f-ae64-bf2db64b661e kind: Event lastTimestamp: "2026-07-20T10:10:21Z" message: Scaled up replica set switch-graph-ab339-7df5ddcb69 from 0 to 1 metadata: creationTimestamp: "2026-07-20T10:10:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-20T10:10:21Z" name: switch-graph-ab339.18c3f7b1f52a1c99 namespace: kserve-ci-e2e-test resourceVersion: "15754" uid: 294c2c59-4e19-417b-8797-608c1fd18f4b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-20T10:10:21Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-ab339 namespace: kserve-ci-e2e-test resourceVersion: "15767" uid: 987e31c6-1c0b-4f2e-9059-2deeb528a35c kind: Event lastTimestamp: "2026-07-20T10:10:21Z" message: InferenceGraph [switch-graph-ab339] is Ready metadata: creationTimestamp: "2026-07-20T10:10:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-20T10:10:21Z" name: switch-graph-ab339.18c3f7b1f995c309 namespace: kserve-ci-e2e-test resourceVersion: "15770" uid: 488d9d36-14ef-49b4-b6b5-7e933a13bba9 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal kind: EventList metadata: resourceVersion: "34528"