--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:39:45.229806Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-6c5e8-5fd6dd9659-55l9b namespace: kserve-ci-e2e-test resourceVersion: "21646" uid: 3a776a68-e7f8-41ba-8639-a81c33ab1bd8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-6c5e8-5fd6dd9659-55l9b to ip-10-0-133-122.ec2.internal metadata: creationTimestamp: "2026-07-02T13:39:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:39:45Z" name: ensemble-graph-6c5e8-5fd6dd9659-55l9b.18be7cad104ab146 namespace: kserve-ci-e2e-test resourceVersion: "21652" uid: 364275ab-7e55-4ec8-99a5-e6a41710460b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7f4bb8bc8-mnq2w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:45Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-6c5e8-5fd6dd9659-55l9b namespace: kserve-ci-e2e-test resourceVersion: "21655" uid: 3a776a68-e7f8-41ba-8639-a81c33ab1bd8 kind: Event lastTimestamp: "2026-07-02T13:39:45Z" message: Add eth0 [10.134.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:39:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:39:45Z" name: ensemble-graph-6c5e8-5fd6dd9659-55l9b.18be7cad2a103439 namespace: kserve-ci-e2e-test resourceVersion: "21667" uid: dbf8446b-262b-456e-9b0d-7c4b1e345c42 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-6c5e8} kind: Pod name: ensemble-graph-6c5e8-5fd6dd9659-55l9b namespace: kserve-ci-e2e-test resourceVersion: "21648" uid: 3a776a68-e7f8-41ba-8639-a81c33ab1bd8 kind: Event lastTimestamp: "2026-07-02T13:39:45Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:9f703da9f2f5afd108b6816a57feadccff9213d4d8437a63e732fa3fbaa3fa9d" already present on machine metadata: creationTimestamp: "2026-07-02T13:39:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:39:45Z" name: ensemble-graph-6c5e8-5fd6dd9659-55l9b.18be7cad2b617db7 namespace: kserve-ci-e2e-test resourceVersion: "21669" uid: 77307e52-0d0d-43d8-a23d-f8db41b37ab5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-6c5e8} kind: Pod name: ensemble-graph-6c5e8-5fd6dd9659-55l9b namespace: kserve-ci-e2e-test resourceVersion: "21648" uid: 3a776a68-e7f8-41ba-8639-a81c33ab1bd8 kind: Event lastTimestamp: "2026-07-02T13:39:45Z" message: 'Created container: ensemble-graph-6c5e8' metadata: creationTimestamp: "2026-07-02T13:39:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:39:45Z" name: ensemble-graph-6c5e8-5fd6dd9659-55l9b.18be7cad2e090ab7 namespace: kserve-ci-e2e-test resourceVersion: "21670" uid: dc7558ad-2b7f-4e11-aef3-160be0cb54c9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-6c5e8} kind: Pod name: ensemble-graph-6c5e8-5fd6dd9659-55l9b namespace: kserve-ci-e2e-test resourceVersion: "21648" uid: 3a776a68-e7f8-41ba-8639-a81c33ab1bd8 kind: Event lastTimestamp: "2026-07-02T13:39:45Z" message: Started container ensemble-graph-6c5e8 metadata: creationTimestamp: "2026-07-02T13:39:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:39:45Z" name: ensemble-graph-6c5e8-5fd6dd9659-55l9b.18be7cad2e79812f namespace: kserve-ci-e2e-test resourceVersion: "21673" uid: 8563f208-4b46-4e9a-9dcb-b09e4ade5e66 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-6c5e8} kind: Pod name: ensemble-graph-6c5e8-5fd6dd9659-55l9b namespace: kserve-ci-e2e-test resourceVersion: "21648" uid: 3a776a68-e7f8-41ba-8639-a81c33ab1bd8 kind: Event lastTimestamp: "2026-07-02T13:39:55Z" message: Stopping container ensemble-graph-6c5e8 metadata: creationTimestamp: "2026-07-02T13:39:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:39:55Z" name: ensemble-graph-6c5e8-5fd6dd9659-55l9b.18be7caf68b09dca namespace: kserve-ci-e2e-test resourceVersion: "21771" uid: e34160eb-69bd-4fec-8f4e-f6c0ad0535c8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:39:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-6c5e8} kind: Pod name: ensemble-graph-6c5e8-5fd6dd9659-55l9b namespace: kserve-ci-e2e-test resourceVersion: "21648" uid: 3a776a68-e7f8-41ba-8639-a81c33ab1bd8 kind: Event lastTimestamp: "2026-07-02T13:40:22Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T13:39:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:40:22Z" name: ensemble-graph-6c5e8-5fd6dd9659-55l9b.18be7cafdc32c136 namespace: kserve-ci-e2e-test resourceVersion: "22131" uid: cc102459-21f6-44f2-84bf-b27dd6720b00 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-6c5e8-5fd6dd9659 namespace: kserve-ci-e2e-test resourceVersion: "21643" uid: 0e9b8922-5faf-4455-9b97-5873e6a7ee48 kind: Event lastTimestamp: "2026-07-02T13:39:45Z" message: 'Created pod: ensemble-graph-6c5e8-5fd6dd9659-55l9b' metadata: creationTimestamp: "2026-07-02T13:39:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:39:45Z" name: ensemble-graph-6c5e8-5fd6dd9659.18be7cad0fe13d3a namespace: kserve-ci-e2e-test resourceVersion: "21647" uid: 5773bfea-2216-41b1-8d1f-9c3ebee1db68 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-6c5e8 namespace: kserve-ci-e2e-test resourceVersion: "21641" uid: 66d28daa-2060-453b-bdcc-f3304cadaff8 kind: Event lastTimestamp: "2026-07-02T13:39:45Z" message: Scaled up replica set ensemble-graph-6c5e8-5fd6dd9659 from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:39:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:39:45Z" name: ensemble-graph-6c5e8.18be7cad0f3fbafc namespace: kserve-ci-e2e-test resourceVersion: "21644" uid: 4fb4a1b2-ef72-4ea0-a05e-c08a0c52cf98 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:45Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-6c5e8 namespace: kserve-ci-e2e-test resourceVersion: "21659" uid: 6c008eee-e07d-443a-a427-54b3985e65c7 kind: Event lastTimestamp: "2026-07-02T13:39:45Z" message: InferenceGraph [ensemble-graph-6c5e8] is Ready metadata: creationTimestamp: "2026-07-02T13:39:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:39:45Z" name: ensemble-graph-6c5e8.18be7cad13ad5f0d namespace: kserve-ci-e2e-test resourceVersion: "21661" uid: 040b9044-ad33-4e9a-a2f8-1887f234b550 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:40:55.497573Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-e2bd2-fc854767f-spdpt namespace: kserve-ci-e2e-test resourceVersion: "22604" uid: b9d58bb1-400e-401d-b130-f384f4fddf79 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-e2bd2-fc854767f-spdpt to ip-10-0-133-122.ec2.internal metadata: creationTimestamp: "2026-07-02T13:40:55Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:40:55Z" name: ensemble-graph-e2bd2-fc854767f-spdpt.18be7cbd6c93c082 namespace: kserve-ci-e2e-test resourceVersion: "22609" uid: 7a8b7db1-5557-4175-90fa-0ec911864e1e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7f4bb8bc8-mnq2w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:40:55Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-e2bd2-fc854767f-spdpt namespace: kserve-ci-e2e-test resourceVersion: "22606" uid: b9d58bb1-400e-401d-b130-f384f4fddf79 kind: Event lastTimestamp: "2026-07-02T13:40:55Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-e2bd2-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T13:40:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:40:55Z" name: ensemble-graph-e2bd2-fc854767f-spdpt.18be7cbd7432c010 namespace: kserve-ci-e2e-test resourceVersion: "22622" uid: a5693531-87df-4d2c-bcef-1faa52c5730f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:40:56Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-e2bd2-fc854767f-spdpt namespace: kserve-ci-e2e-test resourceVersion: "22613" uid: b9d58bb1-400e-401d-b130-f384f4fddf79 kind: Event lastTimestamp: "2026-07-02T13:40:56Z" message: Add eth0 [10.134.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:40:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:40:56Z" name: ensemble-graph-e2bd2-fc854767f-spdpt.18be7cbda9dc7c75 namespace: kserve-ci-e2e-test resourceVersion: "22632" uid: 1ab31203-75fe-4f5c-836a-0def750d653e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:40:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-e2bd2} kind: Pod name: ensemble-graph-e2bd2-fc854767f-spdpt namespace: kserve-ci-e2e-test resourceVersion: "22606" uid: b9d58bb1-400e-401d-b130-f384f4fddf79 kind: Event lastTimestamp: "2026-07-02T13:40:56Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:9f703da9f2f5afd108b6816a57feadccff9213d4d8437a63e732fa3fbaa3fa9d" already present on machine metadata: creationTimestamp: "2026-07-02T13:40:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:40:56Z" name: ensemble-graph-e2bd2-fc854767f-spdpt.18be7cbdab19c85b namespace: kserve-ci-e2e-test resourceVersion: "22635" uid: a7419fd7-40d5-45dc-bd4f-aa7756556fea reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:40:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-e2bd2} kind: Pod name: ensemble-graph-e2bd2-fc854767f-spdpt namespace: kserve-ci-e2e-test resourceVersion: "22606" uid: b9d58bb1-400e-401d-b130-f384f4fddf79 kind: Event lastTimestamp: "2026-07-02T13:40:56Z" message: 'Created container: ensemble-graph-e2bd2' metadata: creationTimestamp: "2026-07-02T13:40:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:40:56Z" name: ensemble-graph-e2bd2-fc854767f-spdpt.18be7cbdae416572 namespace: kserve-ci-e2e-test resourceVersion: "22636" uid: e2163f4a-3c1e-4b0c-bf22-6045b4340ad3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:40:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-e2bd2} kind: Pod name: ensemble-graph-e2bd2-fc854767f-spdpt namespace: kserve-ci-e2e-test resourceVersion: "22606" uid: b9d58bb1-400e-401d-b130-f384f4fddf79 kind: Event lastTimestamp: "2026-07-02T13:40:56Z" message: Started container ensemble-graph-e2bd2 metadata: creationTimestamp: "2026-07-02T13:40:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:40:56Z" name: ensemble-graph-e2bd2-fc854767f-spdpt.18be7cbdaead65d1 namespace: kserve-ci-e2e-test resourceVersion: "22637" uid: 75f0df2f-e8f2-4fd7-8f40-f08c928a7c6d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:49:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-e2bd2} kind: Pod name: ensemble-graph-e2bd2-fc854767f-spdpt namespace: kserve-ci-e2e-test resourceVersion: "22606" uid: b9d58bb1-400e-401d-b130-f384f4fddf79 kind: Event lastTimestamp: "2026-07-02T13:49:15Z" message: Stopping container ensemble-graph-e2bd2 metadata: creationTimestamp: "2026-07-02T13:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:49:15Z" name: ensemble-graph-e2bd2-fc854767f-spdpt.18be7d31c7ef9a32 namespace: kserve-ci-e2e-test resourceVersion: "26448" uid: e34c1563-b621-485e-a0d6-3c025bc3fb6c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:49:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-e2bd2} kind: Pod name: ensemble-graph-e2bd2-fc854767f-spdpt namespace: kserve-ci-e2e-test resourceVersion: "22606" uid: b9d58bb1-400e-401d-b130-f384f4fddf79 kind: Event lastTimestamp: "2026-07-02T13:49:43Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T13:49:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:49:43Z" name: ensemble-graph-e2bd2-fc854767f-spdpt.18be7d328f894ad5 namespace: kserve-ci-e2e-test resourceVersion: "26793" uid: 01865e54-ff21-444f-8803-3bfc23b3538e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:40:55Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-e2bd2-fc854767f namespace: kserve-ci-e2e-test resourceVersion: "22601" uid: 3df63039-f4d0-472c-8964-668251990bb4 kind: Event lastTimestamp: "2026-07-02T13:40:55Z" message: 'Created pod: ensemble-graph-e2bd2-fc854767f-spdpt' metadata: creationTimestamp: "2026-07-02T13:40:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:40:55Z" name: ensemble-graph-e2bd2-fc854767f.18be7cbd6c2d0abd namespace: kserve-ci-e2e-test resourceVersion: "22605" uid: 916ad162-9b10-4f15-9184-abed6a9e258a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:40:55Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-e2bd2 namespace: kserve-ci-e2e-test resourceVersion: "22599" uid: 340cc625-9acf-4790-828b-9b1000081e6c kind: Event lastTimestamp: "2026-07-02T13:40:55Z" message: Scaled up replica set ensemble-graph-e2bd2-fc854767f from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:40:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:40:55Z" name: ensemble-graph-e2bd2.18be7cbd6b9a31b3 namespace: kserve-ci-e2e-test resourceVersion: "22602" uid: 6579f1da-f15d-41a1-ad0a-11a9422c5f86 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:40:55Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-e2bd2 namespace: kserve-ci-e2e-test resourceVersion: "22618" uid: cfca40a0-e78c-4f09-8489-1fe14069618f kind: Event lastTimestamp: "2026-07-02T13:40:55Z" message: InferenceGraph [ensemble-graph-e2bd2] is Ready metadata: creationTimestamp: "2026-07-02T13:40:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:40:55Z" name: ensemble-graph-e2bd2.18be7cbd6fe0b0c9 namespace: kserve-ci-e2e-test resourceVersion: "22620" uid: f0f83aa1-e6a9-43a0-b9a8-a32365c8a06b reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:41:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-e2bd2 namespace: kserve-ci-e2e-test resourceVersion: "22600" uid: fff6435b-ae8e-494f-ba24-a15207cffb96 kind: Event lastTimestamp: "2026-07-02T13:41:10Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:41:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:41:10Z" name: ensemble-graph-e2bd2.18be7cc0e9dbcfbf namespace: kserve-ci-e2e-test resourceVersion: "22762" uid: b7c31b08-7979-47ac-8703-f283bd0d5b12 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:41:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-e2bd2 namespace: kserve-ci-e2e-test resourceVersion: "22600" uid: fff6435b-ae8e-494f-ba24-a15207cffb96 kind: Event lastTimestamp: "2026-07-02T13:41:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:41:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:41:10Z" name: ensemble-graph-e2bd2.18be7cc0ea3444b0 namespace: kserve-ci-e2e-test resourceVersion: "22764" uid: 03c958d9-f665-4708-b09f-6a133b50f66a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:41:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-e2bd2 namespace: kserve-ci-e2e-test resourceVersion: "22763" uid: fff6435b-ae8e-494f-ba24-a15207cffb96 kind: Event lastTimestamp: "2026-07-02T13:41:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:41:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:41:25Z" name: ensemble-graph-e2bd2.18be7cc468b7c09d namespace: kserve-ci-e2e-test resourceVersion: "22885" uid: 6aebe29a-f0f8-47ba-89b8-fd8862281258 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:41:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-e2bd2 namespace: kserve-ci-e2e-test resourceVersion: "22763" uid: fff6435b-ae8e-494f-ba24-a15207cffb96 kind: Event lastTimestamp: "2026-07-02T13:41:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:41:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:41:25Z" name: ensemble-graph-e2bd2.18be7cc46915de88 namespace: kserve-ci-e2e-test resourceVersion: "22887" uid: d2320552-2bb3-44b4-93dd-0b7dc746b664 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:50:25.746472Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-03d0f-predictor-59b6876b74-9wdb4 namespace: kserve-ci-e2e-test resourceVersion: "27419" uid: 1b43aeb7-3760-4843-b59b-718ce64e0011 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-03d0f-predictor-59b6876b74-9wdb4 to ip-10-0-133-122.ec2.internal metadata: creationTimestamp: "2026-07-02T13:50:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:50:25Z" name: error-404-isvc-03d0f-predictor-59b6876b74-9wdb4.18be7d42320f65b5 namespace: kserve-ci-e2e-test resourceVersion: "27423" uid: 6b5753a8-764d-4ed2-b2f4-39be2777fc35 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7f4bb8bc8-mnq2w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:50:25Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-03d0f-predictor-59b6876b74-9wdb4 namespace: kserve-ci-e2e-test resourceVersion: "27421" uid: 1b43aeb7-3760-4843-b59b-718ce64e0011 kind: Event lastTimestamp: "2026-07-02T13:50:25Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-03d0f-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T13:50:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:50:25Z" name: error-404-isvc-03d0f-predictor-59b6876b74-9wdb4.18be7d423a087b53 namespace: kserve-ci-e2e-test resourceVersion: "27450" uid: 861b1a09-c7c0-43e7-af48-864c2c1bdc16 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:50:26Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-03d0f-predictor-59b6876b74-9wdb4 namespace: kserve-ci-e2e-test resourceVersion: "27430" uid: 1b43aeb7-3760-4843-b59b-718ce64e0011 kind: Event lastTimestamp: "2026-07-02T13:50:26Z" message: Add eth0 [10.134.0.47/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:50:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:50:26Z" name: error-404-isvc-03d0f-predictor-59b6876b74-9wdb4.18be7d426f8b6292 namespace: kserve-ci-e2e-test resourceVersion: "27469" uid: 5af25d3b-d6df-4f2f-a527-3b3ef941e908 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:50:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-03d0f-predictor-59b6876b74-9wdb4 namespace: kserve-ci-e2e-test resourceVersion: "27421" uid: 1b43aeb7-3760-4843-b59b-718ce64e0011 kind: Event lastTimestamp: "2026-07-02T13:50:26Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1683" already present on machine metadata: creationTimestamp: "2026-07-02T13:50:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:50:26Z" name: error-404-isvc-03d0f-predictor-59b6876b74-9wdb4.18be7d4270c423ce namespace: kserve-ci-e2e-test resourceVersion: "27471" uid: 90ab1104-563d-48d8-9e33-c57e0dde957b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:50:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-03d0f-predictor-59b6876b74-9wdb4 namespace: kserve-ci-e2e-test resourceVersion: "27421" uid: 1b43aeb7-3760-4843-b59b-718ce64e0011 kind: Event lastTimestamp: "2026-07-02T13:50:26Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T13:50:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:50:26Z" name: error-404-isvc-03d0f-predictor-59b6876b74-9wdb4.18be7d4273d9e758 namespace: kserve-ci-e2e-test resourceVersion: "27473" uid: e48c7f0a-1b16-4508-b1a1-bd472c98e049 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:50:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-03d0f-predictor-59b6876b74-9wdb4 namespace: kserve-ci-e2e-test resourceVersion: "27421" uid: 1b43aeb7-3760-4843-b59b-718ce64e0011 kind: Event lastTimestamp: "2026-07-02T13:50:26Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T13:50:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:50:26Z" name: error-404-isvc-03d0f-predictor-59b6876b74-9wdb4.18be7d42744936b3 namespace: kserve-ci-e2e-test resourceVersion: "27474" uid: 6dcdaa24-4aaa-4816-91c8-b0bfbcbf9186 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:50:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-03d0f-predictor-59b6876b74-9wdb4 namespace: kserve-ci-e2e-test resourceVersion: "27421" uid: 1b43aeb7-3760-4843-b59b-718ce64e0011 kind: Event lastTimestamp: "2026-07-02T13:50:26Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T13:50:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:50:26Z" name: error-404-isvc-03d0f-predictor-59b6876b74-9wdb4.18be7d427452abfd namespace: kserve-ci-e2e-test resourceVersion: "27475" uid: 60c635e2-6e60-42e6-a66f-f2b7785281f1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:50:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-03d0f-predictor-59b6876b74-9wdb4 namespace: kserve-ci-e2e-test resourceVersion: "27421" uid: 1b43aeb7-3760-4843-b59b-718ce64e0011 kind: Event lastTimestamp: "2026-07-02T13:50:26Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T13:50:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:50:26Z" name: error-404-isvc-03d0f-predictor-59b6876b74-9wdb4.18be7d4277c89d5d namespace: kserve-ci-e2e-test resourceVersion: "27476" uid: 0817e97d-ff74-4331-99cb-e88990e2332f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:50:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-03d0f-predictor-59b6876b74-9wdb4 namespace: kserve-ci-e2e-test resourceVersion: "27421" uid: 1b43aeb7-3760-4843-b59b-718ce64e0011 kind: Event lastTimestamp: "2026-07-02T13:50:26Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:50:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:50:26Z" name: error-404-isvc-03d0f-predictor-59b6876b74-9wdb4.18be7d42783f492e namespace: kserve-ci-e2e-test resourceVersion: "27477" uid: 92c121b6-c7f5-42d1-a1f8-93b1a0c0032a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-02T13:50:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-03d0f-predictor-59b6876b74-9wdb4 namespace: kserve-ci-e2e-test resourceVersion: "27421" uid: 1b43aeb7-3760-4843-b59b-718ce64e0011 kind: Event lastTimestamp: "2026-07-02T13:59:43Z" message: 'Readiness probe failed: dial tcp 10.134.0.47:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:50:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:59:43Z" name: error-404-isvc-03d0f-predictor-59b6876b74-9wdb4.18be7d42817bf54d namespace: kserve-ci-e2e-test resourceVersion: "31577" uid: 2ce3d78f-9531-4d84-915a-60448b4fa55b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:59:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-03d0f-predictor-59b6876b74-9wdb4 namespace: kserve-ci-e2e-test resourceVersion: "27421" uid: 1b43aeb7-3760-4843-b59b-718ce64e0011 kind: Event lastTimestamp: "2026-07-02T13:59:40Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T13:59:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:59:40Z" name: error-404-isvc-03d0f-predictor-59b6876b74-9wdb4.18be7dc35d3255e7 namespace: kserve-ci-e2e-test resourceVersion: "31554" uid: a5c6343f-0d23-4145-973d-2e73dcd4b094 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:59:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-03d0f-predictor-59b6876b74-9wdb4 namespace: kserve-ci-e2e-test resourceVersion: "27421" uid: 1b43aeb7-3760-4843-b59b-718ce64e0011 kind: Event lastTimestamp: "2026-07-02T13:59:40Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:59:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:59:40Z" name: error-404-isvc-03d0f-predictor-59b6876b74-9wdb4.18be7dc35d32db31 namespace: kserve-ci-e2e-test resourceVersion: "31555" uid: ef8a82a1-6953-44cc-9e3a-b11d3d5c0397 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:59:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-03d0f-predictor-59b6876b74-9wdb4 namespace: kserve-ci-e2e-test resourceVersion: "27421" uid: 1b43aeb7-3760-4843-b59b-718ce64e0011 kind: Event lastTimestamp: "2026-07-02T13:59:43Z" message: 'Readiness probe failed: Get "https://10.134.0.47:8643/healthz": dial tcp 10.134.0.47:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:59:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:59:43Z" name: error-404-isvc-03d0f-predictor-59b6876b74-9wdb4.18be7dc3f5f0bbd5 namespace: kserve-ci-e2e-test resourceVersion: "31576" uid: 905cb1ce-8e9e-4b1f-a76f-b0ac64cde603 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:50:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-03d0f-predictor-59b6876b74 namespace: kserve-ci-e2e-test resourceVersion: "27411" uid: 513fb110-3a01-4957-b1a6-a9be7d2e4404 kind: Event lastTimestamp: "2026-07-02T13:50:25Z" message: 'Created pod: error-404-isvc-03d0f-predictor-59b6876b74-9wdb4' metadata: creationTimestamp: "2026-07-02T13:50:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:50:25Z" name: error-404-isvc-03d0f-predictor-59b6876b74.18be7d4231aca958 namespace: kserve-ci-e2e-test resourceVersion: "27420" uid: 0c39f15f-0bbd-478a-b4d8-81013e56760d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:50:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-03d0f-predictor namespace: kserve-ci-e2e-test resourceVersion: "27408" uid: 121b6590-bbb5-41fc-9d4a-2329a90a3edd kind: Event lastTimestamp: "2026-07-02T13:50:25Z" message: Scaled up replica set error-404-isvc-03d0f-predictor-59b6876b74 from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:50:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:50:25Z" name: error-404-isvc-03d0f-predictor.18be7d4230464a27 namespace: kserve-ci-e2e-test resourceVersion: "27413" uid: c44157d1-7452-49ca-a609-8abc4ce5b6c3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:50:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-03d0f-predictor namespace: kserve-ci-e2e-test resourceVersion: "27410" uid: fec88f13-a465-473e-867d-0b55539bbdf9 kind: Event lastTimestamp: "2026-07-02T13:50:40Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:50:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:50:40Z" name: error-404-isvc-03d0f-predictor.18be7d45aecd3be8 namespace: kserve-ci-e2e-test resourceVersion: "27628" uid: 7bc431c0-156d-4c1a-b613-e4d248f14d95 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:50:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-03d0f-predictor namespace: kserve-ci-e2e-test resourceVersion: "27410" uid: fec88f13-a465-473e-867d-0b55539bbdf9 kind: Event lastTimestamp: "2026-07-02T13:50:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:50:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:50:40Z" name: error-404-isvc-03d0f-predictor.18be7d45af2a3057 namespace: kserve-ci-e2e-test resourceVersion: "27630" uid: 95198d81-5aaa-4ec8-911b-77b3625937c3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:50:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-03d0f-predictor namespace: kserve-ci-e2e-test resourceVersion: "27629" uid: fec88f13-a465-473e-867d-0b55539bbdf9 kind: Event lastTimestamp: "2026-07-02T13:51:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:50:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:51:25Z" name: error-404-isvc-03d0f-predictor.18be7d492db817c4 namespace: kserve-ci-e2e-test resourceVersion: "28053" uid: a4cdb806-2d58-48ee-b4f2-8601faa7bc41 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:50:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-03d0f-predictor namespace: kserve-ci-e2e-test resourceVersion: "27629" uid: fec88f13-a465-473e-867d-0b55539bbdf9 kind: Event lastTimestamp: "2026-07-02T13:51:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:50:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:51:25Z" name: error-404-isvc-03d0f-predictor.18be7d492e1410dd namespace: kserve-ci-e2e-test resourceVersion: "28056" uid: 67041538-619a-47ba-ba4c-a060fd24296e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:50:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-03d0f namespace: kserve-ci-e2e-test resourceVersion: "27388" uid: 9fd3e5ec-046a-4593-9875-e34518acdced kind: Event lastTimestamp: "2026-07-02T13:51:13Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T13:50:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:51:13Z" name: error-404-isvc-03d0f.18be7d42306f9719 namespace: kserve-ci-e2e-test resourceVersion: "27938" uid: 70dfc7fc-9785-4e7f-8962-43a1460eefa7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:50:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-03d0f namespace: kserve-ci-e2e-test resourceVersion: "27388" uid: 9fd3e5ec-046a-4593-9875-e34518acdced kind: Event lastTimestamp: "2026-07-02T13:50:25Z" message: 'Failed to update status for InferenceService "error-404-isvc-03d0f": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-03d0f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:50:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:50:25Z" name: error-404-isvc-03d0f.18be7d4230bb435b namespace: kserve-ci-e2e-test resourceVersion: "27416" uid: 65b87c17-5723-4df7-8c83-e9f1374e32f6 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:50:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-03d0f namespace: kserve-ci-e2e-test resourceVersion: "27388" uid: 9fd3e5ec-046a-4593-9875-e34518acdced kind: Event lastTimestamp: "2026-07-02T13:50:25Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-03d0f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:50:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:50:25Z" name: error-404-isvc-03d0f.18be7d4230bb9f51 namespace: kserve-ci-e2e-test resourceVersion: "27417" uid: 76c35928-94f0-47a4-b5fd-54aaa0942f9e reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:51:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-03d0f namespace: kserve-ci-e2e-test resourceVersion: "27936" uid: 9fd3e5ec-046a-4593-9875-e34518acdced kind: Event lastTimestamp: "2026-07-02T13:51:13Z" message: InferenceService [error-404-isvc-03d0f] is Ready metadata: creationTimestamp: "2026-07-02T13:51:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:51:13Z" name: error-404-isvc-03d0f.18be7d4d3c3495f8 namespace: kserve-ci-e2e-test resourceVersion: "27937" uid: 212836b0-a939-429f-871f-c7f910b8c2ae reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:39:24.985958Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-2bc9c-predictor-54f54f9cbb-qrbv9 namespace: kserve-ci-e2e-test resourceVersion: "21399" uid: aa940ca5-6df1-4e8f-b8b6-cb9b17811cfc kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-2bc9c-predictor-54f54f9cbb-qrbv9 to ip-10-0-133-122.ec2.internal metadata: creationTimestamp: "2026-07-02T13:39:24Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:39:24Z" name: error-404-isvc-2bc9c-predictor-54f54f9cbb-qrbv9.18be7ca859aa1e9f namespace: kserve-ci-e2e-test resourceVersion: "21403" uid: f3e0fa03-299c-47ba-b229-b5e2916a0056 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7f4bb8bc8-mnq2w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:25Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-2bc9c-predictor-54f54f9cbb-qrbv9 namespace: kserve-ci-e2e-test resourceVersion: "21401" uid: aa940ca5-6df1-4e8f-b8b6-cb9b17811cfc kind: Event lastTimestamp: "2026-07-02T13:39:25Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-2bc9c-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T13:39:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:39:25Z" name: error-404-isvc-2bc9c-predictor-54f54f9cbb-qrbv9.18be7ca86603f973 namespace: kserve-ci-e2e-test resourceVersion: "21419" uid: db7ff617-5cb6-4b18-8e2c-68781e09b334 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:26Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-2bc9c-predictor-54f54f9cbb-qrbv9 namespace: kserve-ci-e2e-test resourceVersion: "21408" uid: aa940ca5-6df1-4e8f-b8b6-cb9b17811cfc kind: Event lastTimestamp: "2026-07-02T13:39:26Z" message: Add eth0 [10.134.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:39:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:39:26Z" name: error-404-isvc-2bc9c-predictor-54f54f9cbb-qrbv9.18be7ca8972f7fc0 namespace: kserve-ci-e2e-test resourceVersion: "21433" uid: 59b9cd55-0eb1-4759-849d-c8e52439a12a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2bc9c-predictor-54f54f9cbb-qrbv9 namespace: kserve-ci-e2e-test resourceVersion: "21401" uid: aa940ca5-6df1-4e8f-b8b6-cb9b17811cfc kind: Event lastTimestamp: "2026-07-02T13:39:26Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1683" already present on machine metadata: creationTimestamp: "2026-07-02T13:39:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:39:26Z" name: error-404-isvc-2bc9c-predictor-54f54f9cbb-qrbv9.18be7ca898dc799e namespace: kserve-ci-e2e-test resourceVersion: "21438" uid: f2db76a3-df15-45af-a9ae-118ccb0ca3e8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2bc9c-predictor-54f54f9cbb-qrbv9 namespace: kserve-ci-e2e-test resourceVersion: "21401" uid: aa940ca5-6df1-4e8f-b8b6-cb9b17811cfc kind: Event lastTimestamp: "2026-07-02T13:39:26Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T13:39:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:39:26Z" name: error-404-isvc-2bc9c-predictor-54f54f9cbb-qrbv9.18be7ca89bc844e6 namespace: kserve-ci-e2e-test resourceVersion: "21441" uid: d20a4f1f-185f-4449-b81c-801aee7b5b92 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2bc9c-predictor-54f54f9cbb-qrbv9 namespace: kserve-ci-e2e-test resourceVersion: "21401" uid: aa940ca5-6df1-4e8f-b8b6-cb9b17811cfc kind: Event lastTimestamp: "2026-07-02T13:39:26Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T13:39:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:39:26Z" name: error-404-isvc-2bc9c-predictor-54f54f9cbb-qrbv9.18be7ca89c387eee namespace: kserve-ci-e2e-test resourceVersion: "21442" uid: a42c9f4e-823b-41ab-ae7f-9cc516d21f5f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2bc9c-predictor-54f54f9cbb-qrbv9 namespace: kserve-ci-e2e-test resourceVersion: "21401" uid: aa940ca5-6df1-4e8f-b8b6-cb9b17811cfc kind: Event lastTimestamp: "2026-07-02T13:39:26Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T13:39:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:39:26Z" name: error-404-isvc-2bc9c-predictor-54f54f9cbb-qrbv9.18be7ca89c42c4f9 namespace: kserve-ci-e2e-test resourceVersion: "21443" uid: 3200f885-b8dd-4908-b1a7-facf5975eaec reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2bc9c-predictor-54f54f9cbb-qrbv9 namespace: kserve-ci-e2e-test resourceVersion: "21401" uid: aa940ca5-6df1-4e8f-b8b6-cb9b17811cfc kind: Event lastTimestamp: "2026-07-02T13:39:26Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T13:39:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:39:26Z" name: error-404-isvc-2bc9c-predictor-54f54f9cbb-qrbv9.18be7ca89fac4709 namespace: kserve-ci-e2e-test resourceVersion: "21444" uid: 5ab34dd8-fb01-45f4-b9b8-9507957bb63d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2bc9c-predictor-54f54f9cbb-qrbv9 namespace: kserve-ci-e2e-test resourceVersion: "21401" uid: aa940ca5-6df1-4e8f-b8b6-cb9b17811cfc kind: Event lastTimestamp: "2026-07-02T13:39:26Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:39:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:39:26Z" name: error-404-isvc-2bc9c-predictor-54f54f9cbb-qrbv9.18be7ca8a01ebd45 namespace: kserve-ci-e2e-test resourceVersion: "21445" uid: 009d2f92-0202-4c06-99fb-0b109a30daef reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:39:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2bc9c-predictor-54f54f9cbb-qrbv9 namespace: kserve-ci-e2e-test resourceVersion: "21401" uid: aa940ca5-6df1-4e8f-b8b6-cb9b17811cfc kind: Event lastTimestamp: "2026-07-02T13:40:04Z" message: 'Readiness probe failed: dial tcp 10.134.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:39:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:40:04Z" name: error-404-isvc-2bc9c-predictor-54f54f9cbb-qrbv9.18be7ca916d06f97 namespace: kserve-ci-e2e-test resourceVersion: "21969" uid: 6a7a78c9-e81f-49b1-bbf6-8c341556248f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:40:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2bc9c-predictor-54f54f9cbb-qrbv9 namespace: kserve-ci-e2e-test resourceVersion: "21401" uid: aa940ca5-6df1-4e8f-b8b6-cb9b17811cfc kind: Event lastTimestamp: "2026-07-02T13:40:35Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T13:40:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:40:35Z" name: error-404-isvc-2bc9c-predictor-54f54f9cbb-qrbv9.18be7cb8aba57dd3 namespace: kserve-ci-e2e-test resourceVersion: "22311" uid: 79f3fb03-c43e-46d2-9c7b-8136825be31c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:40:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2bc9c-predictor-54f54f9cbb-qrbv9 namespace: kserve-ci-e2e-test resourceVersion: "21401" uid: aa940ca5-6df1-4e8f-b8b6-cb9b17811cfc kind: Event lastTimestamp: "2026-07-02T13:40:35Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:40:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:40:35Z" name: error-404-isvc-2bc9c-predictor-54f54f9cbb-qrbv9.18be7cb8aba80817 namespace: kserve-ci-e2e-test resourceVersion: "22314" uid: 511b5ea5-175d-49f3-86c5-61e0145f91f3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:24Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-2bc9c-predictor-54f54f9cbb namespace: kserve-ci-e2e-test resourceVersion: "21390" uid: fc614938-b134-4d7b-9dc9-245b8cd89e8a kind: Event lastTimestamp: "2026-07-02T13:39:24Z" message: 'Created pod: error-404-isvc-2bc9c-predictor-54f54f9cbb-qrbv9' metadata: creationTimestamp: "2026-07-02T13:39:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:39:24Z" name: error-404-isvc-2bc9c-predictor-54f54f9cbb.18be7ca8594b3a93 namespace: kserve-ci-e2e-test resourceVersion: "21400" uid: ebd54dd5-0d18-4f89-b6fa-b98681230e3c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:24Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-2bc9c-predictor namespace: kserve-ci-e2e-test resourceVersion: "21388" uid: ac6d5cba-e378-4410-9c9b-be0cd57139be kind: Event lastTimestamp: "2026-07-02T13:39:24Z" message: Scaled up replica set error-404-isvc-2bc9c-predictor-54f54f9cbb from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:39:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:39:24Z" name: error-404-isvc-2bc9c-predictor.18be7ca857cd4720 namespace: kserve-ci-e2e-test resourceVersion: "21392" uid: 346f9b48-5d6b-4ffe-b8a7-477d63b6c34e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:39:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2bc9c-predictor namespace: kserve-ci-e2e-test resourceVersion: "21389" uid: 9b4deacb-9e42-4f70-b560-9a1fc523afab kind: Event lastTimestamp: "2026-07-02T13:39:54Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:39:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:39:54Z" name: error-404-isvc-2bc9c-predictor.18be7cabd614d13d namespace: kserve-ci-e2e-test resourceVersion: "21752" uid: cd9e67de-3029-43e1-9929-50e71f485b12 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:39:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2bc9c-predictor namespace: kserve-ci-e2e-test resourceVersion: "21389" uid: 9b4deacb-9e42-4f70-b560-9a1fc523afab kind: Event lastTimestamp: "2026-07-02T13:39:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:39:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:39:54Z" name: error-404-isvc-2bc9c-predictor.18be7cabd66f4139 namespace: kserve-ci-e2e-test resourceVersion: "21753" uid: 12661951-62c4-4dac-aeb9-9dc3bd5117fb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:40:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2bc9c-predictor namespace: kserve-ci-e2e-test resourceVersion: "21591" uid: 9b4deacb-9e42-4f70-b560-9a1fc523afab kind: Event lastTimestamp: "2026-07-02T13:40:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:40:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:40:25Z" name: error-404-isvc-2bc9c-predictor.18be7cb2d3db1522 namespace: kserve-ci-e2e-test resourceVersion: "22180" uid: da988833-e123-4903-a886-52ffc914dabf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:40:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2bc9c-predictor namespace: kserve-ci-e2e-test resourceVersion: "21591" uid: 9b4deacb-9e42-4f70-b560-9a1fc523afab kind: Event lastTimestamp: "2026-07-02T13:40:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:40:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:40:25Z" name: error-404-isvc-2bc9c-predictor.18be7cb2d437afdb namespace: kserve-ci-e2e-test resourceVersion: "22182" uid: 148ac275-1a29-429f-9ec2-c4658e251dfc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:39:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2bc9c namespace: kserve-ci-e2e-test resourceVersion: "21372" uid: 60b959ea-a1d6-480a-b226-c9213690ef74 kind: Event lastTimestamp: "2026-07-02T13:40:14Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T13:39:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:40:14Z" name: error-404-isvc-2bc9c.18be7ca857cd9590 namespace: kserve-ci-e2e-test resourceVersion: "22073" uid: 76242ec5-dfba-4424-95c1-3acc6a5e0e1f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2bc9c namespace: kserve-ci-e2e-test resourceVersion: "21372" uid: 60b959ea-a1d6-480a-b226-c9213690ef74 kind: Event lastTimestamp: "2026-07-02T13:39:24Z" message: 'Failed to update status for InferenceService "error-404-isvc-2bc9c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-2bc9c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:39:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:39:24Z" name: error-404-isvc-2bc9c.18be7ca8586f3949 namespace: kserve-ci-e2e-test resourceVersion: "21396" uid: c9993d47-b68f-4122-beba-c461f5a37033 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2bc9c namespace: kserve-ci-e2e-test resourceVersion: "21372" uid: 60b959ea-a1d6-480a-b226-c9213690ef74 kind: Event lastTimestamp: "2026-07-02T13:39:24Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-2bc9c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:39:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:39:24Z" name: error-404-isvc-2bc9c.18be7ca8586f8f9a namespace: kserve-ci-e2e-test resourceVersion: "21397" uid: 330206a3-42d0-4050-af61-5b4f7af0abc2 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:40:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2bc9c namespace: kserve-ci-e2e-test resourceVersion: "22070" uid: 60b959ea-a1d6-480a-b226-c9213690ef74 kind: Event lastTimestamp: "2026-07-02T13:40:14Z" message: InferenceService [error-404-isvc-2bc9c] is Ready metadata: creationTimestamp: "2026-07-02T13:40:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:40:14Z" name: error-404-isvc-2bc9c.18be7cb3d0c7d74a namespace: kserve-ci-e2e-test resourceVersion: "22071" uid: b5774c90-fa03-449a-becc-7086aa18c6dd reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:29:30.333658Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-383ed-predictor-6fbb59466b-2252r namespace: kserve-ci-e2e-test resourceVersion: "16577" uid: ba78be6a-6f3b-419c-97dc-1c86c90721a1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-383ed-predictor-6fbb59466b-2252r to ip-10-0-133-122.ec2.internal metadata: creationTimestamp: "2026-07-02T13:29:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:29:30Z" name: error-404-isvc-383ed-predictor-6fbb59466b-2252r.18be7c1de5a01b19 namespace: kserve-ci-e2e-test resourceVersion: "16581" uid: d5a3a162-0926-4651-aa4b-93c7e43644ec reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7f4bb8bc8-mnq2w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:29:30Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-383ed-predictor-6fbb59466b-2252r namespace: kserve-ci-e2e-test resourceVersion: "16588" uid: ba78be6a-6f3b-419c-97dc-1c86c90721a1 kind: Event lastTimestamp: "2026-07-02T13:29:30Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:29:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:29:30Z" name: error-404-isvc-383ed-predictor-6fbb59466b-2252r.18be7c1dfe9cbea2 namespace: kserve-ci-e2e-test resourceVersion: "16618" uid: b1784058-a899-47c3-8b4b-7030bd0e4183 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:29:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-383ed-predictor-6fbb59466b-2252r namespace: kserve-ci-e2e-test resourceVersion: "16579" uid: ba78be6a-6f3b-419c-97dc-1c86c90721a1 kind: Event lastTimestamp: "2026-07-02T13:29:30Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1683" already present on machine metadata: creationTimestamp: "2026-07-02T13:29:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:29:30Z" name: error-404-isvc-383ed-predictor-6fbb59466b-2252r.18be7c1dffe7f47a namespace: kserve-ci-e2e-test resourceVersion: "16621" uid: 656bd4cd-5b7c-4711-a0bb-c4765a303c5d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:29:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-383ed-predictor-6fbb59466b-2252r namespace: kserve-ci-e2e-test resourceVersion: "16579" uid: ba78be6a-6f3b-419c-97dc-1c86c90721a1 kind: Event lastTimestamp: "2026-07-02T13:29:30Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T13:29:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:29:30Z" name: error-404-isvc-383ed-predictor-6fbb59466b-2252r.18be7c1e025ddf51 namespace: kserve-ci-e2e-test resourceVersion: "16625" uid: 772dc792-d014-4daf-8ed5-07db4ef7c3f8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:29:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-383ed-predictor-6fbb59466b-2252r namespace: kserve-ci-e2e-test resourceVersion: "16579" uid: ba78be6a-6f3b-419c-97dc-1c86c90721a1 kind: Event lastTimestamp: "2026-07-02T13:29:30Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T13:29:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:29:30Z" name: error-404-isvc-383ed-predictor-6fbb59466b-2252r.18be7c1e02ba5e42 namespace: kserve-ci-e2e-test resourceVersion: "16626" uid: a4adf88d-2ecf-41ae-b75d-8af21fb6da71 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:29:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-383ed-predictor-6fbb59466b-2252r namespace: kserve-ci-e2e-test resourceVersion: "16579" uid: ba78be6a-6f3b-419c-97dc-1c86c90721a1 kind: Event lastTimestamp: "2026-07-02T13:29:30Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T13:29:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:29:30Z" name: error-404-isvc-383ed-predictor-6fbb59466b-2252r.18be7c1e02c33ff9 namespace: kserve-ci-e2e-test resourceVersion: "16627" uid: d45cd5b2-602b-41f0-aaf6-d9f8aad6e2e9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:29:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-383ed-predictor-6fbb59466b-2252r namespace: kserve-ci-e2e-test resourceVersion: "16579" uid: ba78be6a-6f3b-419c-97dc-1c86c90721a1 kind: Event lastTimestamp: "2026-07-02T13:29:30Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T13:29:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:29:30Z" name: error-404-isvc-383ed-predictor-6fbb59466b-2252r.18be7c1e0540f4de namespace: kserve-ci-e2e-test resourceVersion: "16631" uid: 7a3c7be5-0764-413c-b465-833639b710a8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:29:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-383ed-predictor-6fbb59466b-2252r namespace: kserve-ci-e2e-test resourceVersion: "16579" uid: ba78be6a-6f3b-419c-97dc-1c86c90721a1 kind: Event lastTimestamp: "2026-07-02T13:29:30Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:29:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:29:30Z" name: error-404-isvc-383ed-predictor-6fbb59466b-2252r.18be7c1e059ed798 namespace: kserve-ci-e2e-test resourceVersion: "16632" uid: 7570b2fc-a923-42de-a503-29e74475db85 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:29:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-383ed-predictor-6fbb59466b-2252r namespace: kserve-ci-e2e-test resourceVersion: "16579" uid: ba78be6a-6f3b-419c-97dc-1c86c90721a1 kind: Event lastTimestamp: "2026-07-02T13:30:08Z" message: 'Readiness probe failed: dial tcp 10.134.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:29:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:30:08Z" name: error-404-isvc-383ed-predictor-6fbb59466b-2252r.18be7c1e6d0707ca namespace: kserve-ci-e2e-test resourceVersion: "16981" uid: 7057af69-6f66-48ad-be9a-23ee2ebcc60f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:38:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-383ed-predictor-6fbb59466b-2252r namespace: kserve-ci-e2e-test resourceVersion: "16579" uid: ba78be6a-6f3b-419c-97dc-1c86c90721a1 kind: Event lastTimestamp: "2026-07-02T13:38:45Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T13:38:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:38:45Z" name: error-404-isvc-383ed-predictor-6fbb59466b-2252r.18be7c9f1b694bd0 namespace: kserve-ci-e2e-test resourceVersion: "20954" uid: ae210e1e-9b71-4723-8790-4e3deca7c9b1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:38:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-383ed-predictor-6fbb59466b-2252r namespace: kserve-ci-e2e-test resourceVersion: "16579" uid: ba78be6a-6f3b-419c-97dc-1c86c90721a1 kind: Event lastTimestamp: "2026-07-02T13:38:45Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:38:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:38:45Z" name: error-404-isvc-383ed-predictor-6fbb59466b-2252r.18be7c9f1b6a8bfa namespace: kserve-ci-e2e-test resourceVersion: "20955" uid: 27d4b85b-c9c1-4435-9f70-586e2beb511d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:29:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-383ed-predictor-6fbb59466b namespace: kserve-ci-e2e-test resourceVersion: "16568" uid: 9a1ab54c-8706-4302-ab4c-c387d7f13627 kind: Event lastTimestamp: "2026-07-02T13:29:30Z" message: 'Created pod: error-404-isvc-383ed-predictor-6fbb59466b-2252r' metadata: creationTimestamp: "2026-07-02T13:29:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:29:30Z" name: error-404-isvc-383ed-predictor-6fbb59466b.18be7c1de5394b1d namespace: kserve-ci-e2e-test resourceVersion: "16578" uid: ef6c1d48-3a4a-4663-928c-c1d6f6920b9f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:29:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-383ed-predictor namespace: kserve-ci-e2e-test resourceVersion: "16565" uid: 1883998b-3a11-4bbe-80c7-b56ede48b751 kind: Event lastTimestamp: "2026-07-02T13:29:30Z" message: Scaled up replica set error-404-isvc-383ed-predictor-6fbb59466b from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:29:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:29:30Z" name: error-404-isvc-383ed-predictor.18be7c1de3c49bbf namespace: kserve-ci-e2e-test resourceVersion: "16571" uid: d3951bfa-4484-4184-b471-5edc88170bc6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:29:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-383ed-predictor namespace: kserve-ci-e2e-test resourceVersion: "16567" uid: 2b09864d-6635-48a6-b212-5546a14690b3 kind: Event lastTimestamp: "2026-07-02T13:30:00Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:29:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:30:00Z" name: error-404-isvc-383ed-predictor.18be7c21622490b8 namespace: kserve-ci-e2e-test resourceVersion: "16915" uid: a1a5422d-12a3-4b79-8a24-b7ccbe2496c1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:29:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-383ed-predictor namespace: kserve-ci-e2e-test resourceVersion: "16567" uid: 2b09864d-6635-48a6-b212-5546a14690b3 kind: Event lastTimestamp: "2026-07-02T13:30:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:29:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:30:00Z" name: error-404-isvc-383ed-predictor.18be7c21628050d7 namespace: kserve-ci-e2e-test resourceVersion: "16916" uid: 12746d79-01f8-48e2-8b2d-6f96dff15018 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:30:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-383ed-predictor namespace: kserve-ci-e2e-test resourceVersion: "16764" uid: 2b09864d-6635-48a6-b212-5546a14690b3 kind: Event lastTimestamp: "2026-07-02T13:30:45Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:30:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:30:45Z" name: error-404-isvc-383ed-predictor.18be7c285f95b3a1 namespace: kserve-ci-e2e-test resourceVersion: "17510" uid: 793df911-ba0b-4f81-a7a3-afb814307c30 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:30:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-383ed-predictor namespace: kserve-ci-e2e-test resourceVersion: "16764" uid: 2b09864d-6635-48a6-b212-5546a14690b3 kind: Event lastTimestamp: "2026-07-02T13:30:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:30:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:30:45Z" name: error-404-isvc-383ed-predictor.18be7c285ff6d584 namespace: kserve-ci-e2e-test resourceVersion: "17511" uid: 9e0a02ed-6023-4ed0-a777-cb829134526f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:29:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-383ed namespace: kserve-ci-e2e-test resourceVersion: "16548" uid: 530a4492-1152-4245-95b7-bd282078b19d kind: Event lastTimestamp: "2026-07-02T13:30:18Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T13:29:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:30:18Z" name: error-404-isvc-383ed.18be7c1de3c619a5 namespace: kserve-ci-e2e-test resourceVersion: "17252" uid: 30a7d22f-63bb-4a41-9502-fdf2ae4eb011 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:29:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-383ed namespace: kserve-ci-e2e-test resourceVersion: "16548" uid: 530a4492-1152-4245-95b7-bd282078b19d kind: Event lastTimestamp: "2026-07-02T13:29:30Z" message: 'Failed to update status for InferenceService "error-404-isvc-383ed": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-383ed": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:29:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:29:30Z" name: error-404-isvc-383ed.18be7c1de4271b72 namespace: kserve-ci-e2e-test resourceVersion: "16574" uid: b9d43481-40c1-4086-bd8f-955db96b4a22 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:29:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-383ed namespace: kserve-ci-e2e-test resourceVersion: "16548" uid: 530a4492-1152-4245-95b7-bd282078b19d kind: Event lastTimestamp: "2026-07-02T13:29:30Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-383ed": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:29:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:29:30Z" name: error-404-isvc-383ed.18be7c1de4274eee namespace: kserve-ci-e2e-test resourceVersion: "16575" uid: 2efe485b-0e9b-49ab-b4b7-c8edd0190d35 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:30:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-383ed namespace: kserve-ci-e2e-test resourceVersion: "17249" uid: 530a4492-1152-4245-95b7-bd282078b19d kind: Event lastTimestamp: "2026-07-02T13:30:18Z" message: InferenceService [error-404-isvc-383ed] is Ready metadata: creationTimestamp: "2026-07-02T13:30:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:30:18Z" name: error-404-isvc-383ed.18be7c29275e0f7d namespace: kserve-ci-e2e-test resourceVersion: "17251" uid: 67960913-a5be-4a9a-9402-dd2fa83cb164 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:38:45.238828Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6c5e8-predictor-bbf5dc8fd-5h8zd namespace: kserve-ci-e2e-test resourceVersion: "20929" uid: fcd366e1-f14f-4f7b-846c-987e5b4b5298 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-6c5e8-predictor-bbf5dc8fd-5h8zd to ip-10-0-133-122.ec2.internal metadata: creationTimestamp: "2026-07-02T13:38:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:38:45Z" name: error-404-isvc-6c5e8-predictor-bbf5dc8fd-5h8zd.18be7c9f188d11de namespace: kserve-ci-e2e-test resourceVersion: "20933" uid: f1a6a8ba-b3f5-4e81-a93c-1080239d4902 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7f4bb8bc8-mnq2w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:38:45Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6c5e8-predictor-bbf5dc8fd-5h8zd namespace: kserve-ci-e2e-test resourceVersion: "20940" uid: fcd366e1-f14f-4f7b-846c-987e5b4b5298 kind: Event lastTimestamp: "2026-07-02T13:38:45Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:38:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:38:45Z" name: error-404-isvc-6c5e8-predictor-bbf5dc8fd-5h8zd.18be7c9f31e4a247 namespace: kserve-ci-e2e-test resourceVersion: "20971" uid: 4a7d66fe-859f-4388-b66e-538231cf1595 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:38:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6c5e8-predictor-bbf5dc8fd-5h8zd namespace: kserve-ci-e2e-test resourceVersion: "20931" uid: fcd366e1-f14f-4f7b-846c-987e5b4b5298 kind: Event lastTimestamp: "2026-07-02T13:38:45Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1683" already present on machine metadata: creationTimestamp: "2026-07-02T13:38:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:38:45Z" name: error-404-isvc-6c5e8-predictor-bbf5dc8fd-5h8zd.18be7c9f332e2f9a namespace: kserve-ci-e2e-test resourceVersion: "20973" uid: da1d97fa-0724-4442-8c72-d399fcea8711 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:38:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6c5e8-predictor-bbf5dc8fd-5h8zd namespace: kserve-ci-e2e-test resourceVersion: "20931" uid: fcd366e1-f14f-4f7b-846c-987e5b4b5298 kind: Event lastTimestamp: "2026-07-02T13:38:45Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T13:38:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:38:45Z" name: error-404-isvc-6c5e8-predictor-bbf5dc8fd-5h8zd.18be7c9f363ddd23 namespace: kserve-ci-e2e-test resourceVersion: "20974" uid: ce05de98-cca1-4d61-9eba-adee7d3e94e2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:38:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6c5e8-predictor-bbf5dc8fd-5h8zd namespace: kserve-ci-e2e-test resourceVersion: "20931" uid: fcd366e1-f14f-4f7b-846c-987e5b4b5298 kind: Event lastTimestamp: "2026-07-02T13:38:45Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T13:38:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:38:45Z" name: error-404-isvc-6c5e8-predictor-bbf5dc8fd-5h8zd.18be7c9f36ab1610 namespace: kserve-ci-e2e-test resourceVersion: "20975" uid: e797129b-a73a-4cd1-9dd5-28515b4b482e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:38:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6c5e8-predictor-bbf5dc8fd-5h8zd namespace: kserve-ci-e2e-test resourceVersion: "20931" uid: fcd366e1-f14f-4f7b-846c-987e5b4b5298 kind: Event lastTimestamp: "2026-07-02T13:38:45Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T13:38:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:38:45Z" name: error-404-isvc-6c5e8-predictor-bbf5dc8fd-5h8zd.18be7c9f36b5bca8 namespace: kserve-ci-e2e-test resourceVersion: "20976" uid: fcb3dbd2-531c-4b15-96a6-48f76a57d13e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:38:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6c5e8-predictor-bbf5dc8fd-5h8zd namespace: kserve-ci-e2e-test resourceVersion: "20931" uid: fcd366e1-f14f-4f7b-846c-987e5b4b5298 kind: Event lastTimestamp: "2026-07-02T13:38:45Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T13:38:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:38:45Z" name: error-404-isvc-6c5e8-predictor-bbf5dc8fd-5h8zd.18be7c9f3a544811 namespace: kserve-ci-e2e-test resourceVersion: "20977" uid: 29a6de7c-0c9d-4590-874a-5a42d4e8849f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:38:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6c5e8-predictor-bbf5dc8fd-5h8zd namespace: kserve-ci-e2e-test resourceVersion: "20931" uid: fcd366e1-f14f-4f7b-846c-987e5b4b5298 kind: Event lastTimestamp: "2026-07-02T13:38:45Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:38:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:38:45Z" name: error-404-isvc-6c5e8-predictor-bbf5dc8fd-5h8zd.18be7c9f3ae21f75 namespace: kserve-ci-e2e-test resourceVersion: "20978" uid: ca163de0-e591-476c-96bd-77dc5b2c25af reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:38:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6c5e8-predictor-bbf5dc8fd-5h8zd namespace: kserve-ci-e2e-test resourceVersion: "20931" uid: fcd366e1-f14f-4f7b-846c-987e5b4b5298 kind: Event lastTimestamp: "2026-07-02T13:39:22Z" message: 'Readiness probe failed: dial tcp 10.134.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:38:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:39:22Z" name: error-404-isvc-6c5e8-predictor-bbf5dc8fd-5h8zd.18be7c9f7f7aaa5c namespace: kserve-ci-e2e-test resourceVersion: "21289" uid: 72dfd911-e26e-447b-b098-512284ffc81b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6c5e8-predictor-bbf5dc8fd-5h8zd namespace: kserve-ci-e2e-test resourceVersion: "20931" uid: fcd366e1-f14f-4f7b-846c-987e5b4b5298 kind: Event lastTimestamp: "2026-07-02T13:39:55Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T13:39:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:39:55Z" name: error-404-isvc-6c5e8-predictor-bbf5dc8fd-5h8zd.18be7caf6d38d314 namespace: kserve-ci-e2e-test resourceVersion: "21792" uid: ce75cbd1-74a2-4efd-aec6-4b5fcccee6d6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6c5e8-predictor-bbf5dc8fd-5h8zd namespace: kserve-ci-e2e-test resourceVersion: "20931" uid: fcd366e1-f14f-4f7b-846c-987e5b4b5298 kind: Event lastTimestamp: "2026-07-02T13:39:55Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:39:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:39:55Z" name: error-404-isvc-6c5e8-predictor-bbf5dc8fd-5h8zd.18be7caf6d397820 namespace: kserve-ci-e2e-test resourceVersion: "21796" uid: e8a1ebe6-652e-4faf-b106-84a594d6cf1c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6c5e8-predictor-bbf5dc8fd-5h8zd namespace: kserve-ci-e2e-test resourceVersion: "20931" uid: fcd366e1-f14f-4f7b-846c-987e5b4b5298 kind: Event lastTimestamp: "2026-07-02T13:39:57Z" message: 'Readiness probe failed: Get "https://10.134.0.36:8643/healthz": dial tcp 10.134.0.36:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:39:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:39:57Z" name: error-404-isvc-6c5e8-predictor-bbf5dc8fd-5h8zd.18be7cb007a87f36 namespace: kserve-ci-e2e-test resourceVersion: "21917" uid: 74da1a77-f581-44e1-9135-bcc52aa94e2d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:38:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-6c5e8-predictor-bbf5dc8fd namespace: kserve-ci-e2e-test resourceVersion: "20920" uid: 80733f15-d4b8-4fbd-80cf-695c748f0da9 kind: Event lastTimestamp: "2026-07-02T13:38:45Z" message: 'Created pod: error-404-isvc-6c5e8-predictor-bbf5dc8fd-5h8zd' metadata: creationTimestamp: "2026-07-02T13:38:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:38:45Z" name: error-404-isvc-6c5e8-predictor-bbf5dc8fd.18be7c9f182c0559 namespace: kserve-ci-e2e-test resourceVersion: "20930" uid: 02a73523-feb2-41c4-a8ec-e8230242bebc reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:38:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-6c5e8-predictor namespace: kserve-ci-e2e-test resourceVersion: "20917" uid: 92e5d26e-cb9c-4c2d-abce-ae9fb08c2dda kind: Event lastTimestamp: "2026-07-02T13:38:45Z" message: Scaled up replica set error-404-isvc-6c5e8-predictor-bbf5dc8fd from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:38:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:38:45Z" name: error-404-isvc-6c5e8-predictor.18be7c9f16d22914 namespace: kserve-ci-e2e-test resourceVersion: "20923" uid: 1d53044b-baa9-461c-b12e-f9dd43a646ee reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:39:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6c5e8-predictor namespace: kserve-ci-e2e-test resourceVersion: "20919" uid: bab73b6c-70ab-44e2-a33b-2d4820817275 kind: Event lastTimestamp: "2026-07-02T13:39:15Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:39:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:39:15Z" name: error-404-isvc-6c5e8-predictor.18be7ca2952ff8ff namespace: kserve-ci-e2e-test resourceVersion: "21229" uid: bbb7030a-6d8c-4a18-a230-c91011ee9bef reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:39:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6c5e8-predictor namespace: kserve-ci-e2e-test resourceVersion: "20919" uid: bab73b6c-70ab-44e2-a33b-2d4820817275 kind: Event lastTimestamp: "2026-07-02T13:39:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:39:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:39:15Z" name: error-404-isvc-6c5e8-predictor.18be7ca2958ea381 namespace: kserve-ci-e2e-test resourceVersion: "21230" uid: b47d781e-27ac-4854-936a-344577aea683 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:39:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6c5e8-predictor namespace: kserve-ci-e2e-test resourceVersion: "21118" uid: bab73b6c-70ab-44e2-a33b-2d4820817275 kind: Event lastTimestamp: "2026-07-02T13:39:45Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:39:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:39:45Z" name: error-404-isvc-6c5e8-predictor.18be7ca992b96e9a namespace: kserve-ci-e2e-test resourceVersion: "21656" uid: 02c73247-d011-4cd8-a9f9-dd402e8ed96d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:39:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6c5e8-predictor namespace: kserve-ci-e2e-test resourceVersion: "21118" uid: bab73b6c-70ab-44e2-a33b-2d4820817275 kind: Event lastTimestamp: "2026-07-02T13:39:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:39:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:39:45Z" name: error-404-isvc-6c5e8-predictor.18be7ca993159f3b namespace: kserve-ci-e2e-test resourceVersion: "21657" uid: ff8a07b4-2d64-4833-bc07-2fd323b3b1c5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:38:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6c5e8 namespace: kserve-ci-e2e-test resourceVersion: "20903" uid: c8ac77c0-cc06-4ee7-9898-4931546416bb kind: Event lastTimestamp: "2026-07-02T13:39:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T13:38:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:39:33Z" name: error-404-isvc-6c5e8.18be7c9f16d75762 namespace: kserve-ci-e2e-test resourceVersion: "21532" uid: 393ced91-a455-49bf-bbf0-acc47bf5a684 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:38:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6c5e8 namespace: kserve-ci-e2e-test resourceVersion: "20903" uid: c8ac77c0-cc06-4ee7-9898-4931546416bb kind: Event lastTimestamp: "2026-07-02T13:38:45Z" message: 'Failed to update status for InferenceService "error-404-isvc-6c5e8": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-6c5e8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:38:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:38:45Z" name: error-404-isvc-6c5e8.18be7c9f17339d53 namespace: kserve-ci-e2e-test resourceVersion: "20926" uid: c5ceae3a-5e45-4195-a2bb-f0d30aeaf532 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:38:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6c5e8 namespace: kserve-ci-e2e-test resourceVersion: "20903" uid: c8ac77c0-cc06-4ee7-9898-4931546416bb kind: Event lastTimestamp: "2026-07-02T13:38:45Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-6c5e8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:38:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:38:45Z" name: error-404-isvc-6c5e8.18be7c9f17340e02 namespace: kserve-ci-e2e-test resourceVersion: "20927" uid: 542c91c0-94fc-4f2b-907a-ab751b95f210 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6c5e8 namespace: kserve-ci-e2e-test resourceVersion: "21530" uid: c8ac77c0-cc06-4ee7-9898-4931546416bb kind: Event lastTimestamp: "2026-07-02T13:39:33Z" message: InferenceService [error-404-isvc-6c5e8] is Ready metadata: creationTimestamp: "2026-07-02T13:39:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:39:33Z" name: error-404-isvc-6c5e8.18be7caa39dbd4a4 namespace: kserve-ci-e2e-test resourceVersion: "21531" uid: f18a0265-0db1-4015-b44c-d8cb6b1c9311 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:40:35.210228Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7b15c-predictor-6c577469b4-7vqmd namespace: kserve-ci-e2e-test resourceVersion: "22364" uid: 70a269e0-b1a9-40d8-b7a7-c4bdce124a68 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-7b15c-predictor-6c577469b4-7vqmd to ip-10-0-133-122.ec2.internal metadata: creationTimestamp: "2026-07-02T13:40:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:40:35Z" name: error-404-isvc-7b15c-predictor-6c577469b4-7vqmd.18be7cb8b35b681d namespace: kserve-ci-e2e-test resourceVersion: "22368" uid: fc7f09aa-f87f-4bfd-a5cb-fb89defd7a00 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7f4bb8bc8-mnq2w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:40:35Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7b15c-predictor-6c577469b4-7vqmd namespace: kserve-ci-e2e-test resourceVersion: "22374" uid: 70a269e0-b1a9-40d8-b7a7-c4bdce124a68 kind: Event lastTimestamp: "2026-07-02T13:40:35Z" message: Add eth0 [10.134.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:40:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:40:35Z" name: error-404-isvc-7b15c-predictor-6c577469b4-7vqmd.18be7cb8ccc63789 namespace: kserve-ci-e2e-test resourceVersion: "22389" uid: d1c7817b-b39c-4fa9-8ca7-35ad947b5567 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:40:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7b15c-predictor-6c577469b4-7vqmd namespace: kserve-ci-e2e-test resourceVersion: "22366" uid: 70a269e0-b1a9-40d8-b7a7-c4bdce124a68 kind: Event lastTimestamp: "2026-07-02T13:40:35Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1683" already present on machine metadata: creationTimestamp: "2026-07-02T13:40:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:40:35Z" name: error-404-isvc-7b15c-predictor-6c577469b4-7vqmd.18be7cb8ce1301e3 namespace: kserve-ci-e2e-test resourceVersion: "22394" uid: 53536ef8-f4f2-467e-b0bf-2d6613fd2bae reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:40:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7b15c-predictor-6c577469b4-7vqmd namespace: kserve-ci-e2e-test resourceVersion: "22366" uid: 70a269e0-b1a9-40d8-b7a7-c4bdce124a68 kind: Event lastTimestamp: "2026-07-02T13:40:35Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T13:40:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:40:35Z" name: error-404-isvc-7b15c-predictor-6c577469b4-7vqmd.18be7cb8d1389619 namespace: kserve-ci-e2e-test resourceVersion: "22397" uid: 9c83fd90-f4a6-4da8-bd52-e3b68eaa793c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:40:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7b15c-predictor-6c577469b4-7vqmd namespace: kserve-ci-e2e-test resourceVersion: "22366" uid: 70a269e0-b1a9-40d8-b7a7-c4bdce124a68 kind: Event lastTimestamp: "2026-07-02T13:40:35Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T13:40:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:40:35Z" name: error-404-isvc-7b15c-predictor-6c577469b4-7vqmd.18be7cb8d1a8e2fc namespace: kserve-ci-e2e-test resourceVersion: "22398" uid: 38e3bdae-1d1e-40ca-94a5-e8337d1ecc5e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:40:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7b15c-predictor-6c577469b4-7vqmd namespace: kserve-ci-e2e-test resourceVersion: "22366" uid: 70a269e0-b1a9-40d8-b7a7-c4bdce124a68 kind: Event lastTimestamp: "2026-07-02T13:40:35Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T13:40:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:40:35Z" name: error-404-isvc-7b15c-predictor-6c577469b4-7vqmd.18be7cb8d1d2c3ac namespace: kserve-ci-e2e-test resourceVersion: "22399" uid: 4a36baed-4e56-404a-ae7d-8a9390574653 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:40:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7b15c-predictor-6c577469b4-7vqmd namespace: kserve-ci-e2e-test resourceVersion: "22366" uid: 70a269e0-b1a9-40d8-b7a7-c4bdce124a68 kind: Event lastTimestamp: "2026-07-02T13:40:35Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T13:40:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:40:35Z" name: error-404-isvc-7b15c-predictor-6c577469b4-7vqmd.18be7cb8d47718d0 namespace: kserve-ci-e2e-test resourceVersion: "22400" uid: e4731da1-56a4-443c-8a5d-8ef116b15555 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:40:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7b15c-predictor-6c577469b4-7vqmd namespace: kserve-ci-e2e-test resourceVersion: "22366" uid: 70a269e0-b1a9-40d8-b7a7-c4bdce124a68 kind: Event lastTimestamp: "2026-07-02T13:40:35Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:40:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:40:35Z" name: error-404-isvc-7b15c-predictor-6c577469b4-7vqmd.18be7cb8d4e75e64 namespace: kserve-ci-e2e-test resourceVersion: "22401" uid: 62e195d5-5eda-4298-a3eb-9607c1806fcf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:40:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7b15c-predictor-6c577469b4-7vqmd namespace: kserve-ci-e2e-test resourceVersion: "22366" uid: 70a269e0-b1a9-40d8-b7a7-c4bdce124a68 kind: Event lastTimestamp: "2026-07-02T13:41:13Z" message: 'Readiness probe failed: dial tcp 10.134.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:40:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:41:13Z" name: error-404-isvc-7b15c-predictor-6c577469b4-7vqmd.18be7cb93b8d8186 namespace: kserve-ci-e2e-test resourceVersion: "22784" uid: cea8ac29-392b-46d6-8435-93f018eeaa29 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:49:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7b15c-predictor-6c577469b4-7vqmd namespace: kserve-ci-e2e-test resourceVersion: "22366" uid: 70a269e0-b1a9-40d8-b7a7-c4bdce124a68 kind: Event lastTimestamp: "2026-07-02T13:49:50Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T13:49:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:49:50Z" name: error-404-isvc-7b15c-predictor-6c577469b4-7vqmd.18be7d39e19bea6b namespace: kserve-ci-e2e-test resourceVersion: "26890" uid: 9848f57d-b103-4f6e-921d-f8294ff0e359 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:49:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7b15c-predictor-6c577469b4-7vqmd namespace: kserve-ci-e2e-test resourceVersion: "22366" uid: 70a269e0-b1a9-40d8-b7a7-c4bdce124a68 kind: Event lastTimestamp: "2026-07-02T13:49:50Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:49:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:49:50Z" name: error-404-isvc-7b15c-predictor-6c577469b4-7vqmd.18be7d39e19c6d34 namespace: kserve-ci-e2e-test resourceVersion: "26895" uid: b34ddc05-fd39-40c9-ad10-f0788435a312 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:49:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7b15c-predictor-6c577469b4-7vqmd namespace: kserve-ci-e2e-test resourceVersion: "22366" uid: 70a269e0-b1a9-40d8-b7a7-c4bdce124a68 kind: Event lastTimestamp: "2026-07-02T13:49:53Z" message: 'Readiness probe failed: Get "https://10.134.0.41:8643/healthz": dial tcp 10.134.0.41:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:49:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:49:53Z" name: error-404-isvc-7b15c-predictor-6c577469b4-7vqmd.18be7d3ab01490da namespace: kserve-ci-e2e-test resourceVersion: "27018" uid: e13a2cb5-17d6-44c0-9335-334c2860989d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:49:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7b15c-predictor-6c577469b4-7vqmd namespace: kserve-ci-e2e-test resourceVersion: "22366" uid: 70a269e0-b1a9-40d8-b7a7-c4bdce124a68 kind: Event lastTimestamp: "2026-07-02T13:49:54Z" message: 'Readiness probe failed: dial tcp 10.134.0.41:8080: i/o timeout' metadata: creationTimestamp: "2026-07-02T13:49:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:49:54Z" name: error-404-isvc-7b15c-predictor-6c577469b4-7vqmd.18be7d3aec161454 namespace: kserve-ci-e2e-test resourceVersion: "27030" uid: 7f7a810a-948e-453b-b937-1c1281305e75 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:40:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-7b15c-predictor-6c577469b4 namespace: kserve-ci-e2e-test resourceVersion: "22356" uid: 1dbfce46-2c2f-4e96-9ec2-2dc7c5e7f2b4 kind: Event lastTimestamp: "2026-07-02T13:40:35Z" message: 'Created pod: error-404-isvc-7b15c-predictor-6c577469b4-7vqmd' metadata: creationTimestamp: "2026-07-02T13:40:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:40:35Z" name: error-404-isvc-7b15c-predictor-6c577469b4.18be7cb8b2d97a7f namespace: kserve-ci-e2e-test resourceVersion: "22365" uid: bfb3343d-1cc6-4d27-80fa-6cffce0007bb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:40:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-7b15c-predictor namespace: kserve-ci-e2e-test resourceVersion: "22353" uid: 4e146757-1489-43ad-863b-1c2f799a2cd7 kind: Event lastTimestamp: "2026-07-02T13:40:35Z" message: Scaled up replica set error-404-isvc-7b15c-predictor-6c577469b4 from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:40:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:40:35Z" name: error-404-isvc-7b15c-predictor.18be7cb8b147f2e6 namespace: kserve-ci-e2e-test resourceVersion: "22357" uid: 100097fb-9635-489f-ae1f-5fbe9ff80fd7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:40:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7b15c-predictor namespace: kserve-ci-e2e-test resourceVersion: "22354" uid: 54fa5833-d7cd-4271-a686-bf69964d87e0 kind: Event lastTimestamp: "2026-07-02T13:41:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:40:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:41:20Z" name: error-404-isvc-7b15c-predictor.18be7cbc2fab99e3 namespace: kserve-ci-e2e-test resourceVersion: "22832" uid: d0d82d7f-fa31-441d-90c8-b77f0f603bc6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:40:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7b15c-predictor namespace: kserve-ci-e2e-test resourceVersion: "22354" uid: 54fa5833-d7cd-4271-a686-bf69964d87e0 kind: Event lastTimestamp: "2026-07-02T13:41:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:40:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:41:20Z" name: error-404-isvc-7b15c-predictor.18be7cbc2ffe9b5f namespace: kserve-ci-e2e-test resourceVersion: "22834" uid: 1830df6b-cedb-476d-aa46-ec1e3c453b0c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:41:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7b15c-predictor namespace: kserve-ci-e2e-test resourceVersion: "22556" uid: 54fa5833-d7cd-4271-a686-bf69964d87e0 kind: Event lastTimestamp: "2026-07-02T13:41:50Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:41:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:41:50Z" name: error-404-isvc-7b15c-predictor.18be7cbfae8d7267 namespace: kserve-ci-e2e-test resourceVersion: "23106" uid: 729ffc4e-c876-4c8b-8c86-498e4167e2da reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:41:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7b15c-predictor namespace: kserve-ci-e2e-test resourceVersion: "22556" uid: 54fa5833-d7cd-4271-a686-bf69964d87e0 kind: Event lastTimestamp: "2026-07-02T13:41:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:41:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:41:50Z" name: error-404-isvc-7b15c-predictor.18be7cbfaeeb8922 namespace: kserve-ci-e2e-test resourceVersion: "23107" uid: 47d283d9-054e-4fcc-b070-39e0540404a7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:40:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7b15c namespace: kserve-ci-e2e-test resourceVersion: "22333" uid: 0ba8ce10-acaf-4b00-895a-f4b848ce0501 kind: Event lastTimestamp: "2026-07-02T13:41:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T13:40:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:41:23Z" name: error-404-isvc-7b15c.18be7cb8b148e5ea namespace: kserve-ci-e2e-test resourceVersion: "22871" uid: f36b4568-a50c-44c5-87dc-aefcdd1d2288 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:40:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7b15c namespace: kserve-ci-e2e-test resourceVersion: "22333" uid: 0ba8ce10-acaf-4b00-895a-f4b848ce0501 kind: Event lastTimestamp: "2026-07-02T13:40:35Z" message: 'Failed to update status for InferenceService "error-404-isvc-7b15c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-7b15c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:40:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:40:35Z" name: error-404-isvc-7b15c.18be7cb8b19f4633 namespace: kserve-ci-e2e-test resourceVersion: "22361" uid: e5c9383b-3251-48db-bce2-b2ab47b63351 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:40:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7b15c namespace: kserve-ci-e2e-test resourceVersion: "22333" uid: 0ba8ce10-acaf-4b00-895a-f4b848ce0501 kind: Event lastTimestamp: "2026-07-02T13:40:35Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-7b15c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:40:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:40:35Z" name: error-404-isvc-7b15c.18be7cb8b19f8cd9 namespace: kserve-ci-e2e-test resourceVersion: "22362" uid: 6e48f4e5-84b2-44cb-93f7-8496dd83a20a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:41:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7b15c namespace: kserve-ci-e2e-test resourceVersion: "22869" uid: 0ba8ce10-acaf-4b00-895a-f4b848ce0501 kind: Event lastTimestamp: "2026-07-02T13:41:23Z" message: InferenceService [error-404-isvc-7b15c] is Ready metadata: creationTimestamp: "2026-07-02T13:41:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:41:23Z" name: error-404-isvc-7b15c.18be7cc3f60488bc namespace: kserve-ci-e2e-test resourceVersion: "22870" uid: 151a5df7-6a9f-470b-8800-f05761437023 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:49:50.170730Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7f184-predictor-5c9f6d84c6-srwjk namespace: kserve-ci-e2e-test resourceVersion: "26942" uid: 09ac4a1b-aff2-4e55-9f19-87b60d5ad4a7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-7f184-predictor-5c9f6d84c6-srwjk to ip-10-0-133-122.ec2.internal metadata: creationTimestamp: "2026-07-02T13:49:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:49:50Z" name: error-404-isvc-7f184-predictor-5c9f6d84c6-srwjk.18be7d39e994a20b namespace: kserve-ci-e2e-test resourceVersion: "26946" uid: 3854da63-5f2d-4db9-bd73-f21502963da5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7f4bb8bc8-mnq2w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:49:50Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7f184-predictor-5c9f6d84c6-srwjk namespace: kserve-ci-e2e-test resourceVersion: "26944" uid: 09ac4a1b-aff2-4e55-9f19-87b60d5ad4a7 kind: Event lastTimestamp: "2026-07-02T13:49:50Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-7f184-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T13:49:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:49:50Z" name: error-404-isvc-7f184-predictor-5c9f6d84c6-srwjk.18be7d39f769d6e0 namespace: kserve-ci-e2e-test resourceVersion: "26960" uid: 25cdc902-22ff-456f-ac02-571e2a1b3ecd reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:49:51Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7f184-predictor-5c9f6d84c6-srwjk namespace: kserve-ci-e2e-test resourceVersion: "26952" uid: 09ac4a1b-aff2-4e55-9f19-87b60d5ad4a7 kind: Event lastTimestamp: "2026-07-02T13:49:51Z" message: Add eth0 [10.134.0.45/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:49:51Z" name: error-404-isvc-7f184-predictor-5c9f6d84c6-srwjk.18be7d3a26e4e135 namespace: kserve-ci-e2e-test resourceVersion: "26975" uid: b0917922-13b9-4e2f-b15b-ae8a39a5d6cc reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:49:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7f184-predictor-5c9f6d84c6-srwjk namespace: kserve-ci-e2e-test resourceVersion: "26944" uid: 09ac4a1b-aff2-4e55-9f19-87b60d5ad4a7 kind: Event lastTimestamp: "2026-07-02T13:49:51Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1683" already present on machine metadata: creationTimestamp: "2026-07-02T13:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:49:51Z" name: error-404-isvc-7f184-predictor-5c9f6d84c6-srwjk.18be7d3a2841751c namespace: kserve-ci-e2e-test resourceVersion: "26980" uid: 0ee6bbe5-0367-492a-8c10-902d949e73d8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:49:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7f184-predictor-5c9f6d84c6-srwjk namespace: kserve-ci-e2e-test resourceVersion: "26944" uid: 09ac4a1b-aff2-4e55-9f19-87b60d5ad4a7 kind: Event lastTimestamp: "2026-07-02T13:49:51Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T13:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:49:51Z" name: error-404-isvc-7f184-predictor-5c9f6d84c6-srwjk.18be7d3a2b09e8c9 namespace: kserve-ci-e2e-test resourceVersion: "26981" uid: 1a2495c9-dc71-4fea-a676-59c80b9af62e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:49:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7f184-predictor-5c9f6d84c6-srwjk namespace: kserve-ci-e2e-test resourceVersion: "26944" uid: 09ac4a1b-aff2-4e55-9f19-87b60d5ad4a7 kind: Event lastTimestamp: "2026-07-02T13:49:51Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T13:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:49:51Z" name: error-404-isvc-7f184-predictor-5c9f6d84c6-srwjk.18be7d3a2b76696e namespace: kserve-ci-e2e-test resourceVersion: "26983" uid: 3eb0f00e-f5b7-41a7-8d61-9d471566c3cb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:49:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7f184-predictor-5c9f6d84c6-srwjk namespace: kserve-ci-e2e-test resourceVersion: "26944" uid: 09ac4a1b-aff2-4e55-9f19-87b60d5ad4a7 kind: Event lastTimestamp: "2026-07-02T13:49:51Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T13:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:49:51Z" name: error-404-isvc-7f184-predictor-5c9f6d84c6-srwjk.18be7d3a2b7ef3c3 namespace: kserve-ci-e2e-test resourceVersion: "26985" uid: 7e526bd3-bfb6-4c1f-a95c-17886453c954 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:49:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7f184-predictor-5c9f6d84c6-srwjk namespace: kserve-ci-e2e-test resourceVersion: "26944" uid: 09ac4a1b-aff2-4e55-9f19-87b60d5ad4a7 kind: Event lastTimestamp: "2026-07-02T13:49:51Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T13:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:49:51Z" name: error-404-isvc-7f184-predictor-5c9f6d84c6-srwjk.18be7d3a2ec5611c namespace: kserve-ci-e2e-test resourceVersion: "26986" uid: f9fd6720-b297-416a-bc2a-76acd18979a2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:49:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7f184-predictor-5c9f6d84c6-srwjk namespace: kserve-ci-e2e-test resourceVersion: "26944" uid: 09ac4a1b-aff2-4e55-9f19-87b60d5ad4a7 kind: Event lastTimestamp: "2026-07-02T13:49:51Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:49:51Z" name: error-404-isvc-7f184-predictor-5c9f6d84c6-srwjk.18be7d3a2f578ca2 namespace: kserve-ci-e2e-test resourceVersion: "26987" uid: fc68fa7c-2c7a-473e-81ef-a608ef72e97c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:49:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7f184-predictor-5c9f6d84c6-srwjk namespace: kserve-ci-e2e-test resourceVersion: "26944" uid: 09ac4a1b-aff2-4e55-9f19-87b60d5ad4a7 kind: Event lastTimestamp: "2026-07-02T13:50:28Z" message: 'Readiness probe failed: dial tcp 10.134.0.45:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:49:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:50:28Z" name: error-404-isvc-7f184-predictor-5c9f6d84c6-srwjk.18be7d3a8d5abe4c namespace: kserve-ci-e2e-test resourceVersion: "27504" uid: 83267568-1491-43b9-a648-b57f2f96a707 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:07:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7f184-predictor-5c9f6d84c6-srwjk namespace: kserve-ci-e2e-test resourceVersion: "26944" uid: 09ac4a1b-aff2-4e55-9f19-87b60d5ad4a7 kind: Event lastTimestamp: "2026-07-02T14:07:09Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:07:09Z" name: error-404-isvc-7f184-predictor-5c9f6d84c6-srwjk.18be7e2beeb61f5a namespace: kserve-ci-e2e-test resourceVersion: "34661" uid: f010fd1f-24d7-4eab-bc49-1c2440a8d085 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:07:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7f184-predictor-5c9f6d84c6-srwjk namespace: kserve-ci-e2e-test resourceVersion: "26944" uid: 09ac4a1b-aff2-4e55-9f19-87b60d5ad4a7 kind: Event lastTimestamp: "2026-07-02T14:07:09Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:07:09Z" name: error-404-isvc-7f184-predictor-5c9f6d84c6-srwjk.18be7e2beeb6d380 namespace: kserve-ci-e2e-test resourceVersion: "34662" uid: a6d360db-5564-434b-9002-ae5957e0cd05 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:49:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-7f184-predictor-5c9f6d84c6 namespace: kserve-ci-e2e-test resourceVersion: "26934" uid: e892bd47-ed6d-4f07-ac1b-26a18350d28b kind: Event lastTimestamp: "2026-07-02T13:49:50Z" message: 'Created pod: error-404-isvc-7f184-predictor-5c9f6d84c6-srwjk' metadata: creationTimestamp: "2026-07-02T13:49:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:49:50Z" name: error-404-isvc-7f184-predictor-5c9f6d84c6.18be7d39e90bae30 namespace: kserve-ci-e2e-test resourceVersion: "26943" uid: 294076b5-4468-403d-ab1e-c2bcd3b62f83 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:49:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-7f184-predictor namespace: kserve-ci-e2e-test resourceVersion: "26931" uid: e2e94c4f-d357-4c2f-89aa-fbeec20ebbb7 kind: Event lastTimestamp: "2026-07-02T13:49:50Z" message: Scaled up replica set error-404-isvc-7f184-predictor-5c9f6d84c6 from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:49:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:49:50Z" name: error-404-isvc-7f184-predictor.18be7d39e79b90c4 namespace: kserve-ci-e2e-test resourceVersion: "26935" uid: fc971cfd-f00d-4e51-939e-41bc474b25b2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:50:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7f184-predictor namespace: kserve-ci-e2e-test resourceVersion: "26933" uid: 77985b90-4aa1-4ff6-ac8d-e9c4e1b851ce kind: Event lastTimestamp: "2026-07-02T13:50:35Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:50:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:50:35Z" name: error-404-isvc-7f184-predictor.18be7d3d65ec3c17 namespace: kserve-ci-e2e-test resourceVersion: "27561" uid: 4cdfcb1f-f31d-405a-92e2-2aeffaa38e9e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:50:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7f184-predictor namespace: kserve-ci-e2e-test resourceVersion: "26933" uid: 77985b90-4aa1-4ff6-ac8d-e9c4e1b851ce kind: Event lastTimestamp: "2026-07-02T13:50:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:50:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:50:35Z" name: error-404-isvc-7f184-predictor.18be7d3d664a9706 namespace: kserve-ci-e2e-test resourceVersion: "27562" uid: f2b4e46c-28f9-4e02-93c5-db414ff09b33 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:50:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7f184-predictor namespace: kserve-ci-e2e-test resourceVersion: "27135" uid: 77985b90-4aa1-4ff6-ac8d-e9c4e1b851ce kind: Event lastTimestamp: "2026-07-02T13:51:05Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:50:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:51:05Z" name: error-404-isvc-7f184-predictor.18be7d40e4d01eea namespace: kserve-ci-e2e-test resourceVersion: "27865" uid: 1dd7a85a-b2f3-4080-bafb-b5ce4b85cae1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:50:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7f184-predictor namespace: kserve-ci-e2e-test resourceVersion: "27135" uid: 77985b90-4aa1-4ff6-ac8d-e9c4e1b851ce kind: Event lastTimestamp: "2026-07-02T13:51:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:50:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:51:05Z" name: error-404-isvc-7f184-predictor.18be7d40e52f038d namespace: kserve-ci-e2e-test resourceVersion: "27866" uid: 840e4c18-85cd-4450-b696-5771774d39c3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:49:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7f184 namespace: kserve-ci-e2e-test resourceVersion: "26914" uid: 9f0ddc88-3f8b-4d42-b4f3-cdb4956cd180 kind: Event lastTimestamp: "2026-07-02T13:50:39Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T13:49:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:50:39Z" name: error-404-isvc-7f184.18be7d39e7a304c8 namespace: kserve-ci-e2e-test resourceVersion: "27611" uid: b7ca288e-4926-4272-af2c-a5f82cf56ccd reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:49:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7f184 namespace: kserve-ci-e2e-test resourceVersion: "26914" uid: 9f0ddc88-3f8b-4d42-b4f3-cdb4956cd180 kind: Event lastTimestamp: "2026-07-02T13:49:50Z" message: 'Failed to update status for InferenceService "error-404-isvc-7f184": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-7f184": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:49:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:49:50Z" name: error-404-isvc-7f184.18be7d39e8177663 namespace: kserve-ci-e2e-test resourceVersion: "26938" uid: 99b584e9-1393-480a-aaed-877d6e0d668d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:49:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7f184 namespace: kserve-ci-e2e-test resourceVersion: "26914" uid: 9f0ddc88-3f8b-4d42-b4f3-cdb4956cd180 kind: Event lastTimestamp: "2026-07-02T13:49:50Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-7f184": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:49:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:49:50Z" name: error-404-isvc-7f184.18be7d39e817aafa namespace: kserve-ci-e2e-test resourceVersion: "26939" uid: 5195fa50-c09a-461a-98ce-af57e87d1e2c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:50:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7f184 namespace: kserve-ci-e2e-test resourceVersion: "27609" uid: 9f0ddc88-3f8b-4d42-b4f3-cdb4956cd180 kind: Event lastTimestamp: "2026-07-02T13:50:38Z" message: InferenceService [error-404-isvc-7f184] is Ready metadata: creationTimestamp: "2026-07-02T13:50:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:50:38Z" name: error-404-isvc-7f184.18be7d4547951a81 namespace: kserve-ci-e2e-test resourceVersion: "27610" uid: 33d035be-8b5d-4615-8b4c-86923ca20283 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:49:15.475480Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-9633b-predictor-6db75d69f8-hx5s5 namespace: kserve-ci-e2e-test resourceVersion: "26517" uid: 2df728f4-3c51-46d9-bcf9-40f3da845c0a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-9633b-predictor-6db75d69f8-hx5s5 to ip-10-0-133-122.ec2.internal metadata: creationTimestamp: "2026-07-02T13:49:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:49:15Z" name: error-404-isvc-9633b-predictor-6db75d69f8-hx5s5.18be7d31d5952891 namespace: kserve-ci-e2e-test resourceVersion: "26522" uid: 56bf8585-cd24-486b-939b-65ffe94d35fe reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7f4bb8bc8-mnq2w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:49:15Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-9633b-predictor-6db75d69f8-hx5s5 namespace: kserve-ci-e2e-test resourceVersion: "26527" uid: 2df728f4-3c51-46d9-bcf9-40f3da845c0a kind: Event lastTimestamp: "2026-07-02T13:49:15Z" message: Add eth0 [10.134.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:49:15Z" name: error-404-isvc-9633b-predictor-6db75d69f8-hx5s5.18be7d31ef25e1fe namespace: kserve-ci-e2e-test resourceVersion: "26557" uid: 92c4a274-0088-4771-8b2a-c08e6612eba6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:49:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9633b-predictor-6db75d69f8-hx5s5 namespace: kserve-ci-e2e-test resourceVersion: "26519" uid: 2df728f4-3c51-46d9-bcf9-40f3da845c0a kind: Event lastTimestamp: "2026-07-02T13:49:15Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1683" already present on machine metadata: creationTimestamp: "2026-07-02T13:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:49:15Z" name: error-404-isvc-9633b-predictor-6db75d69f8-hx5s5.18be7d31f08b0bbd namespace: kserve-ci-e2e-test resourceVersion: "26560" uid: f8c86a90-ab68-49c4-b295-efc09dafea8e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:49:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9633b-predictor-6db75d69f8-hx5s5 namespace: kserve-ci-e2e-test resourceVersion: "26519" uid: 2df728f4-3c51-46d9-bcf9-40f3da845c0a kind: Event lastTimestamp: "2026-07-02T13:49:15Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T13:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:49:15Z" name: error-404-isvc-9633b-predictor-6db75d69f8-hx5s5.18be7d31f376e628 namespace: kserve-ci-e2e-test resourceVersion: "26564" uid: 9eadb53d-16c3-438e-82d7-3ea58c188751 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:49:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9633b-predictor-6db75d69f8-hx5s5 namespace: kserve-ci-e2e-test resourceVersion: "26519" uid: 2df728f4-3c51-46d9-bcf9-40f3da845c0a kind: Event lastTimestamp: "2026-07-02T13:49:15Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T13:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:49:15Z" name: error-404-isvc-9633b-predictor-6db75d69f8-hx5s5.18be7d31f3e53d81 namespace: kserve-ci-e2e-test resourceVersion: "26565" uid: 95915211-5d98-4c57-a503-c289d1e4c948 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:49:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9633b-predictor-6db75d69f8-hx5s5 namespace: kserve-ci-e2e-test resourceVersion: "26519" uid: 2df728f4-3c51-46d9-bcf9-40f3da845c0a kind: Event lastTimestamp: "2026-07-02T13:49:15Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T13:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:49:15Z" name: error-404-isvc-9633b-predictor-6db75d69f8-hx5s5.18be7d31f3ee95d0 namespace: kserve-ci-e2e-test resourceVersion: "26567" uid: 757257b4-297b-4357-bc0c-77afb70a118c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:49:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9633b-predictor-6db75d69f8-hx5s5 namespace: kserve-ci-e2e-test resourceVersion: "26519" uid: 2df728f4-3c51-46d9-bcf9-40f3da845c0a kind: Event lastTimestamp: "2026-07-02T13:49:16Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T13:49:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:49:16Z" name: error-404-isvc-9633b-predictor-6db75d69f8-hx5s5.18be7d31f6c09479 namespace: kserve-ci-e2e-test resourceVersion: "26569" uid: 93f34309-e468-4fd5-88cd-b3e377f71cc6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:49:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9633b-predictor-6db75d69f8-hx5s5 namespace: kserve-ci-e2e-test resourceVersion: "26519" uid: 2df728f4-3c51-46d9-bcf9-40f3da845c0a kind: Event lastTimestamp: "2026-07-02T13:49:16Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:49:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:49:16Z" name: error-404-isvc-9633b-predictor-6db75d69f8-hx5s5.18be7d31f73339fc namespace: kserve-ci-e2e-test resourceVersion: "26570" uid: f3d9fc0b-0bac-4b25-8965-7810a27dafae reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:49:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9633b-predictor-6db75d69f8-hx5s5 namespace: kserve-ci-e2e-test resourceVersion: "26519" uid: 2df728f4-3c51-46d9-bcf9-40f3da845c0a kind: Event lastTimestamp: "2026-07-02T13:49:52Z" message: 'Readiness probe failed: dial tcp 10.134.0.44:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:49:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:49:52Z" name: error-404-isvc-9633b-predictor-6db75d69f8-hx5s5.18be7d322166c8bb namespace: kserve-ci-e2e-test resourceVersion: "27007" uid: bf28524f-d80e-4c71-a99d-c5c960eeb939 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:50:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9633b-predictor-6db75d69f8-hx5s5 namespace: kserve-ci-e2e-test resourceVersion: "26519" uid: 2df728f4-3c51-46d9-bcf9-40f3da845c0a kind: Event lastTimestamp: "2026-07-02T13:50:25Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T13:50:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:50:25Z" name: error-404-isvc-9633b-predictor-6db75d69f8-hx5s5.18be7d42355dc0cd namespace: kserve-ci-e2e-test resourceVersion: "27444" uid: faed4e2a-c2a4-463a-a3be-0571f16c2e52 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:50:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9633b-predictor-6db75d69f8-hx5s5 namespace: kserve-ci-e2e-test resourceVersion: "26519" uid: 2df728f4-3c51-46d9-bcf9-40f3da845c0a kind: Event lastTimestamp: "2026-07-02T13:50:25Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:50:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:50:25Z" name: error-404-isvc-9633b-predictor-6db75d69f8-hx5s5.18be7d42355e3a27 namespace: kserve-ci-e2e-test resourceVersion: "27446" uid: a69baa40-79d5-46e5-ba87-bacc541f8425 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:50:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9633b-predictor-6db75d69f8-hx5s5 namespace: kserve-ci-e2e-test resourceVersion: "26519" uid: 2df728f4-3c51-46d9-bcf9-40f3da845c0a kind: Event lastTimestamp: "2026-07-02T13:50:27Z" message: 'Readiness probe failed: Get "https://10.134.0.44:8643/healthz": dial tcp 10.134.0.44:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:50:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:50:27Z" name: error-404-isvc-9633b-predictor-6db75d69f8-hx5s5.18be7d42a98dab20 namespace: kserve-ci-e2e-test resourceVersion: "27490" uid: ac225ae9-7ec1-4484-aa00-171d2485c4e8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:49:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-9633b-predictor-6db75d69f8 namespace: kserve-ci-e2e-test resourceVersion: "26503" uid: 1f617ed0-1d1a-4211-999a-8ff6b036d84e kind: Event lastTimestamp: "2026-07-02T13:49:15Z" message: 'Created pod: error-404-isvc-9633b-predictor-6db75d69f8-hx5s5' metadata: creationTimestamp: "2026-07-02T13:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:49:15Z" name: error-404-isvc-9633b-predictor-6db75d69f8.18be7d31d5118f5d namespace: kserve-ci-e2e-test resourceVersion: "26518" uid: cab7d145-4d19-4ffb-b336-cd6d043d85e0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:49:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-9633b-predictor namespace: kserve-ci-e2e-test resourceVersion: "26497" uid: b51493d2-10cc-4f17-8369-9c2f2953e215 kind: Event lastTimestamp: "2026-07-02T13:49:15Z" message: Scaled up replica set error-404-isvc-9633b-predictor-6db75d69f8 from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:49:15Z" name: error-404-isvc-9633b-predictor.18be7d31d35cf790 namespace: kserve-ci-e2e-test resourceVersion: "26507" uid: a108fffd-a157-456a-acd6-ee1b8cf8388b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:49:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9633b-predictor namespace: kserve-ci-e2e-test resourceVersion: "26501" uid: c5ed3a80-4fe6-4acd-baf5-5e13461a4859 kind: Event lastTimestamp: "2026-07-02T13:49:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:49:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:49:45Z" name: error-404-isvc-9633b-predictor.18be7d3551ea16f7 namespace: kserve-ci-e2e-test resourceVersion: "26809" uid: 15276112-f86b-45d9-bffa-3cc0b45223c8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:49:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9633b-predictor namespace: kserve-ci-e2e-test resourceVersion: "26501" uid: c5ed3a80-4fe6-4acd-baf5-5e13461a4859 kind: Event lastTimestamp: "2026-07-02T13:49:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:49:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:49:45Z" name: error-404-isvc-9633b-predictor.18be7d355241f988 namespace: kserve-ci-e2e-test resourceVersion: "26810" uid: ac38d2a2-1d78-4b0d-91f5-a75360c0f222 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:50:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9633b-predictor namespace: kserve-ci-e2e-test resourceVersion: "26697" uid: c5ed3a80-4fe6-4acd-baf5-5e13461a4859 kind: Event lastTimestamp: "2026-07-02T13:50:15Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:50:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:50:15Z" name: error-404-isvc-9633b-predictor.18be7d3c4f5a17ee namespace: kserve-ci-e2e-test resourceVersion: "27237" uid: 5fc601b4-bb9c-433e-8ec1-bf828f5ded18 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:50:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9633b-predictor namespace: kserve-ci-e2e-test resourceVersion: "26697" uid: c5ed3a80-4fe6-4acd-baf5-5e13461a4859 kind: Event lastTimestamp: "2026-07-02T13:50:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:50:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:50:15Z" name: error-404-isvc-9633b-predictor.18be7d3c4fb6d86d namespace: kserve-ci-e2e-test resourceVersion: "27238" uid: 3c24ad3b-1ce8-43a4-8e8c-b22e7ffa18de reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T13:49:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9633b namespace: kserve-ci-e2e-test resourceVersion: "26474" uid: 82ea2e81-dde7-4664-b2f2-b6403881118e kind: Event lastTimestamp: "2026-07-02T13:50:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T13:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:50:02Z" name: error-404-isvc-9633b.18be7d31d3ab944e namespace: kserve-ci-e2e-test resourceVersion: "27105" uid: 25dd726b-d6c3-4b32-b2d5-f10745046619 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:49:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9633b namespace: kserve-ci-e2e-test resourceVersion: "26474" uid: 82ea2e81-dde7-4664-b2f2-b6403881118e kind: Event lastTimestamp: "2026-07-02T13:49:15Z" message: 'Failed to update status for InferenceService "error-404-isvc-9633b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-9633b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:49:15Z" name: error-404-isvc-9633b.18be7d31d410e8b9 namespace: kserve-ci-e2e-test resourceVersion: "26512" uid: a3d5c849-ddcc-43ef-baaf-e8144e6e70a2 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:49:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9633b namespace: kserve-ci-e2e-test resourceVersion: "26474" uid: 82ea2e81-dde7-4664-b2f2-b6403881118e kind: Event lastTimestamp: "2026-07-02T13:49:15Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-9633b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:49:15Z" name: error-404-isvc-9633b.18be7d31d411259f namespace: kserve-ci-e2e-test resourceVersion: "26516" uid: dcfd8de2-d222-4b49-9d39-2d711a160765 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:50:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9633b namespace: kserve-ci-e2e-test resourceVersion: "27103" uid: 82ea2e81-dde7-4664-b2f2-b6403881118e kind: Event lastTimestamp: "2026-07-02T13:50:02Z" message: InferenceService [error-404-isvc-9633b] is Ready metadata: creationTimestamp: "2026-07-02T13:50:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:50:02Z" name: error-404-isvc-9633b.18be7d3cdb9b9687 namespace: kserve-ci-e2e-test resourceVersion: "27104" uid: c6eab2ac-86f7-4b1d-9d92-fb7d984d324b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:27:40.150902Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-acf44-predictor-7bdc57d7d4-bsv9h namespace: kserve-ci-e2e-test resourceVersion: "15353" uid: 7b1b08cc-9488-4eb1-ab3b-6358ea27439a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-acf44-predictor-7bdc57d7d4-bsv9h to ip-10-0-133-122.ec2.internal metadata: creationTimestamp: "2026-07-02T13:27:40Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:27:40Z" name: error-404-isvc-acf44-predictor-7bdc57d7d4-bsv9h.18be7c043e38a51a namespace: kserve-ci-e2e-test resourceVersion: "15357" uid: 8dce019f-9123-4c2c-b2fd-8816c6364176 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7f4bb8bc8-mnq2w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:27:40Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-acf44-predictor-7bdc57d7d4-bsv9h namespace: kserve-ci-e2e-test resourceVersion: "15362" uid: 7b1b08cc-9488-4eb1-ab3b-6358ea27439a kind: Event lastTimestamp: "2026-07-02T13:27:40Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:27:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:27:40Z" name: error-404-isvc-acf44-predictor-7bdc57d7d4-bsv9h.18be7c0457ddb2c0 namespace: kserve-ci-e2e-test resourceVersion: "15436" uid: 6a0111c1-2b6d-49bf-86d6-a2839cc09a84 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:27:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-acf44-predictor-7bdc57d7d4-bsv9h namespace: kserve-ci-e2e-test resourceVersion: "15355" uid: 7b1b08cc-9488-4eb1-ab3b-6358ea27439a kind: Event lastTimestamp: "2026-07-02T13:27:40Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1683" metadata: creationTimestamp: "2026-07-02T13:27:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:27:40Z" name: error-404-isvc-acf44-predictor-7bdc57d7d4-bsv9h.18be7c04591cf1b8 namespace: kserve-ci-e2e-test resourceVersion: "15439" uid: a5314d57-bad9-4e90-b0e0-bc89b9238abe reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:27:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-acf44-predictor-7bdc57d7d4-bsv9h namespace: kserve-ci-e2e-test resourceVersion: "15355" uid: 7b1b08cc-9488-4eb1-ab3b-6358ea27439a kind: Event lastTimestamp: "2026-07-02T13:27:52Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1683" in 12.142s (12.142s including waiting). Image size: 1334623835 bytes.' metadata: creationTimestamp: "2026-07-02T13:27:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:27:52Z" name: error-404-isvc-acf44-predictor-7bdc57d7d4-bsv9h.18be7c072cd974ea namespace: kserve-ci-e2e-test resourceVersion: "15574" uid: 061847af-3dc9-4bdb-99d4-e8bc5fe918f3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:27:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-acf44-predictor-7bdc57d7d4-bsv9h namespace: kserve-ci-e2e-test resourceVersion: "15355" uid: 7b1b08cc-9488-4eb1-ab3b-6358ea27439a kind: Event lastTimestamp: "2026-07-02T13:27:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T13:27:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:27:52Z" name: error-404-isvc-acf44-predictor-7bdc57d7d4-bsv9h.18be7c072fc3c36e namespace: kserve-ci-e2e-test resourceVersion: "15575" uid: 9ccfcdee-33fe-453a-88b9-f069413012c5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:27:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-acf44-predictor-7bdc57d7d4-bsv9h namespace: kserve-ci-e2e-test resourceVersion: "15355" uid: 7b1b08cc-9488-4eb1-ab3b-6358ea27439a kind: Event lastTimestamp: "2026-07-02T13:27:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T13:27:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:27:52Z" name: error-404-isvc-acf44-predictor-7bdc57d7d4-bsv9h.18be7c0730337ddd namespace: kserve-ci-e2e-test resourceVersion: "15576" uid: 4828dbd6-6a40-413d-ace4-5381695870d6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:27:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-acf44-predictor-7bdc57d7d4-bsv9h namespace: kserve-ci-e2e-test resourceVersion: "15355" uid: 7b1b08cc-9488-4eb1-ab3b-6358ea27439a kind: Event lastTimestamp: "2026-07-02T13:27:52Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-02T13:27:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:27:52Z" name: error-404-isvc-acf44-predictor-7bdc57d7d4-bsv9h.18be7c0730417042 namespace: kserve-ci-e2e-test resourceVersion: "15577" uid: 3f94f6f4-96f1-48c4-b001-189f6170444b reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:27:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-acf44-predictor-7bdc57d7d4-bsv9h namespace: kserve-ci-e2e-test resourceVersion: "15355" uid: 7b1b08cc-9488-4eb1-ab3b-6358ea27439a kind: Event lastTimestamp: "2026-07-02T13:27:55Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.289s (2.289s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-02T13:27:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:27:55Z" name: error-404-isvc-acf44-predictor-7bdc57d7d4-bsv9h.18be7c07b8bbbbf4 namespace: kserve-ci-e2e-test resourceVersion: "15595" uid: 57cb06c0-0d88-4549-905f-ccd951fabbb2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:27:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-acf44-predictor-7bdc57d7d4-bsv9h namespace: kserve-ci-e2e-test resourceVersion: "15355" uid: 7b1b08cc-9488-4eb1-ab3b-6358ea27439a kind: Event lastTimestamp: "2026-07-02T13:27:55Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T13:27:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:27:55Z" name: error-404-isvc-acf44-predictor-7bdc57d7d4-bsv9h.18be7c07bc02cacc namespace: kserve-ci-e2e-test resourceVersion: "15600" uid: 3f29e4ef-4540-40b1-bbaa-216425f11436 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:27:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-acf44-predictor-7bdc57d7d4-bsv9h namespace: kserve-ci-e2e-test resourceVersion: "15355" uid: 7b1b08cc-9488-4eb1-ab3b-6358ea27439a kind: Event lastTimestamp: "2026-07-02T13:27:55Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:27:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:27:55Z" name: error-404-isvc-acf44-predictor-7bdc57d7d4-bsv9h.18be7c07bc77dc98 namespace: kserve-ci-e2e-test resourceVersion: "15601" uid: 9bb0a9ec-fe3e-44e4-84d3-b77cd5cf971b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-02T13:27:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-acf44-predictor-7bdc57d7d4-bsv9h namespace: kserve-ci-e2e-test resourceVersion: "15355" uid: 7b1b08cc-9488-4eb1-ab3b-6358ea27439a kind: Event lastTimestamp: "2026-07-02T13:29:32Z" message: 'Readiness probe failed: dial tcp 10.134.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:27:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:29:32Z" name: error-404-isvc-acf44-predictor-7bdc57d7d4-bsv9h.18be7c07fb2d4b80 namespace: kserve-ci-e2e-test resourceVersion: "16651" uid: 624d598d-996a-4795-8a0a-b031d0995c7a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:29:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-acf44-predictor-7bdc57d7d4-bsv9h namespace: kserve-ci-e2e-test resourceVersion: "15355" uid: 7b1b08cc-9488-4eb1-ab3b-6358ea27439a kind: Event lastTimestamp: "2026-07-02T13:29:30Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T13:29:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:29:30Z" name: error-404-isvc-acf44-predictor-7bdc57d7d4-bsv9h.18be7c1de9075f49 namespace: kserve-ci-e2e-test resourceVersion: "16603" uid: f4118d63-41d5-4cdd-b7c9-a362f174a071 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:29:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-acf44-predictor-7bdc57d7d4-bsv9h namespace: kserve-ci-e2e-test resourceVersion: "15355" uid: 7b1b08cc-9488-4eb1-ab3b-6358ea27439a kind: Event lastTimestamp: "2026-07-02T13:29:30Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:29:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:29:30Z" name: error-404-isvc-acf44-predictor-7bdc57d7d4-bsv9h.18be7c1de9080aed namespace: kserve-ci-e2e-test resourceVersion: "16604" uid: b31f06d1-85fd-4037-a089-39573dd810da reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:29:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-acf44-predictor-7bdc57d7d4-bsv9h namespace: kserve-ci-e2e-test resourceVersion: "15355" uid: 7b1b08cc-9488-4eb1-ab3b-6358ea27439a kind: Event lastTimestamp: "2026-07-02T13:29:32Z" message: 'Readiness probe failed: Get "https://10.134.0.29:8643/healthz": dial tcp 10.134.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:29:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:29:32Z" name: error-404-isvc-acf44-predictor-7bdc57d7d4-bsv9h.18be7c1e556c67f0 namespace: kserve-ci-e2e-test resourceVersion: "16650" uid: e1ade2e2-7a2a-4e15-9b7b-ab2ca496dff5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:27:40Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-acf44-predictor-7bdc57d7d4 namespace: kserve-ci-e2e-test resourceVersion: "15345" uid: f9c8fe35-0eb7-445a-b5f0-288c9a57aaa8 kind: Event lastTimestamp: "2026-07-02T13:27:40Z" message: 'Created pod: error-404-isvc-acf44-predictor-7bdc57d7d4-bsv9h' metadata: creationTimestamp: "2026-07-02T13:27:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:27:40Z" name: error-404-isvc-acf44-predictor-7bdc57d7d4.18be7c043dd63740 namespace: kserve-ci-e2e-test resourceVersion: "15354" uid: 26d8916e-afce-4a56-9b11-1cbd61268391 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:27:40Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-acf44-predictor namespace: kserve-ci-e2e-test resourceVersion: "15343" uid: dc8914ed-f611-4475-8877-8a0544d50916 kind: Event lastTimestamp: "2026-07-02T13:27:40Z" message: Scaled up replica set error-404-isvc-acf44-predictor-7bdc57d7d4 from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:27:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:27:40Z" name: error-404-isvc-acf44-predictor.18be7c043a8fe9ba namespace: kserve-ci-e2e-test resourceVersion: "15346" uid: 5eca9226-08c6-483e-b929-1546304a1c1a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:27:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-acf44-predictor namespace: kserve-ci-e2e-test resourceVersion: "15344" uid: 96d6e029-066f-488b-82dc-3e44bac30274 kind: Event lastTimestamp: "2026-07-02T13:28:25Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:27:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:28:25Z" name: error-404-isvc-acf44-predictor.18be7c07b8fb1bdb namespace: kserve-ci-e2e-test resourceVersion: "15890" uid: 802fb747-d28d-4317-8a2e-baf6c2b44132 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:27:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-acf44-predictor namespace: kserve-ci-e2e-test resourceVersion: "15344" uid: 96d6e029-066f-488b-82dc-3e44bac30274 kind: Event lastTimestamp: "2026-07-02T13:28:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:27:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:28:25Z" name: error-404-isvc-acf44-predictor.18be7c07b95260e2 namespace: kserve-ci-e2e-test resourceVersion: "15891" uid: e5585265-be11-4322-a775-f18b2dbc4472 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:28:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-acf44-predictor namespace: kserve-ci-e2e-test resourceVersion: "15597" uid: 96d6e029-066f-488b-82dc-3e44bac30274 kind: Event lastTimestamp: "2026-07-02T13:29:10Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:28:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:29:10Z" name: error-404-isvc-acf44-predictor.18be7c12351e9c50 namespace: kserve-ci-e2e-test resourceVersion: "16268" uid: f4140040-4677-4e17-9cce-bd57875c15ab reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:28:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-acf44-predictor namespace: kserve-ci-e2e-test resourceVersion: "15597" uid: 96d6e029-066f-488b-82dc-3e44bac30274 kind: Event lastTimestamp: "2026-07-02T13:29:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:28:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:29:10Z" name: error-404-isvc-acf44-predictor.18be7c123577777c namespace: kserve-ci-e2e-test resourceVersion: "16269" uid: eb927fad-b152-476f-beec-e2acab971a5c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T13:27:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-acf44 namespace: kserve-ci-e2e-test resourceVersion: "15327" uid: 8df7261a-23b4-4dfe-bc1f-e492b798ec68 kind: Event lastTimestamp: "2026-07-02T13:28:42Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T13:27:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:28:42Z" name: error-404-isvc-acf44.18be7c043a91bdb3 namespace: kserve-ci-e2e-test resourceVersion: "16046" uid: 094c0081-7e52-4539-b5a1-71530ac9098b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:28:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-acf44 namespace: kserve-ci-e2e-test resourceVersion: "16044" uid: 8df7261a-23b4-4dfe-bc1f-e492b798ec68 kind: Event lastTimestamp: "2026-07-02T13:28:42Z" message: InferenceService [error-404-isvc-acf44] is Ready metadata: creationTimestamp: "2026-07-02T13:28:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:28:42Z" name: error-404-isvc-acf44.18be7c12b59f1beb namespace: kserve-ci-e2e-test resourceVersion: "16045" uid: f883b02f-b638-4a5e-ab19-f1fa47410828 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:39:55.530016Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e2bd2-predictor-677686d8bb-mxljv namespace: kserve-ci-e2e-test resourceVersion: "21850" uid: e000058e-9278-448d-8993-fd666a913d9e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-e2bd2-predictor-677686d8bb-mxljv to ip-10-0-133-122.ec2.internal metadata: creationTimestamp: "2026-07-02T13:39:55Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:39:55Z" name: error-404-isvc-e2bd2-predictor-677686d8bb-mxljv.18be7caf763b6ff4 namespace: kserve-ci-e2e-test resourceVersion: "21854" uid: 1e43a29c-6e3f-4abb-8d23-68d8cfdfabb6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7f4bb8bc8-mnq2w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:55Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e2bd2-predictor-677686d8bb-mxljv namespace: kserve-ci-e2e-test resourceVersion: "21852" uid: e000058e-9278-448d-8993-fd666a913d9e kind: Event lastTimestamp: "2026-07-02T13:39:55Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-e2bd2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T13:39:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:39:55Z" name: error-404-isvc-e2bd2-predictor-677686d8bb-mxljv.18be7caf82998dfe namespace: kserve-ci-e2e-test resourceVersion: "21869" uid: 83c0bb8c-f302-4cbc-a6b0-44a871a0a839 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:56Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e2bd2-predictor-677686d8bb-mxljv namespace: kserve-ci-e2e-test resourceVersion: "21861" uid: e000058e-9278-448d-8993-fd666a913d9e kind: Event lastTimestamp: "2026-07-02T13:39:56Z" message: Add eth0 [10.134.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:39:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:39:56Z" name: error-404-isvc-e2bd2-predictor-677686d8bb-mxljv.18be7cafb3dcd685 namespace: kserve-ci-e2e-test resourceVersion: "21887" uid: e0960311-3cae-4592-b8a9-6221e5eb6a10 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e2bd2-predictor-677686d8bb-mxljv namespace: kserve-ci-e2e-test resourceVersion: "21852" uid: e000058e-9278-448d-8993-fd666a913d9e kind: Event lastTimestamp: "2026-07-02T13:39:56Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1683" already present on machine metadata: creationTimestamp: "2026-07-02T13:39:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:39:56Z" name: error-404-isvc-e2bd2-predictor-677686d8bb-mxljv.18be7cafb5318fb7 namespace: kserve-ci-e2e-test resourceVersion: "21889" uid: 9a3b28ba-dbd6-46cf-9871-c14de4987475 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e2bd2-predictor-677686d8bb-mxljv namespace: kserve-ci-e2e-test resourceVersion: "21852" uid: e000058e-9278-448d-8993-fd666a913d9e kind: Event lastTimestamp: "2026-07-02T13:39:56Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T13:39:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:39:56Z" name: error-404-isvc-e2bd2-predictor-677686d8bb-mxljv.18be7cafb96724d4 namespace: kserve-ci-e2e-test resourceVersion: "21892" uid: d4bc7052-4836-4766-a26a-b36a304349b7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e2bd2-predictor-677686d8bb-mxljv namespace: kserve-ci-e2e-test resourceVersion: "21852" uid: e000058e-9278-448d-8993-fd666a913d9e kind: Event lastTimestamp: "2026-07-02T13:39:56Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T13:39:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:39:56Z" name: error-404-isvc-e2bd2-predictor-677686d8bb-mxljv.18be7cafb9dcbca5 namespace: kserve-ci-e2e-test resourceVersion: "21893" uid: 9b6d65e0-da53-4862-b272-3f4f74733235 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e2bd2-predictor-677686d8bb-mxljv namespace: kserve-ci-e2e-test resourceVersion: "21852" uid: e000058e-9278-448d-8993-fd666a913d9e kind: Event lastTimestamp: "2026-07-02T13:39:56Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T13:39:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:39:56Z" name: error-404-isvc-e2bd2-predictor-677686d8bb-mxljv.18be7cafb9e7f765 namespace: kserve-ci-e2e-test resourceVersion: "21894" uid: d81bdb87-f0fa-4c3b-9878-5592c852ed31 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e2bd2-predictor-677686d8bb-mxljv namespace: kserve-ci-e2e-test resourceVersion: "21852" uid: e000058e-9278-448d-8993-fd666a913d9e kind: Event lastTimestamp: "2026-07-02T13:39:56Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T13:39:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:39:56Z" name: error-404-isvc-e2bd2-predictor-677686d8bb-mxljv.18be7cafbd3bd7e8 namespace: kserve-ci-e2e-test resourceVersion: "21895" uid: ff02202b-c957-451d-9e96-dab59529ca47 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e2bd2-predictor-677686d8bb-mxljv namespace: kserve-ci-e2e-test resourceVersion: "21852" uid: e000058e-9278-448d-8993-fd666a913d9e kind: Event lastTimestamp: "2026-07-02T13:39:56Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:39:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:39:56Z" name: error-404-isvc-e2bd2-predictor-677686d8bb-mxljv.18be7cafbdc4e07f namespace: kserve-ci-e2e-test resourceVersion: "21897" uid: 3d3d9bf1-054f-4043-930a-15fb1062bf90 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:39:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e2bd2-predictor-677686d8bb-mxljv namespace: kserve-ci-e2e-test resourceVersion: "21852" uid: e000058e-9278-448d-8993-fd666a913d9e kind: Event lastTimestamp: "2026-07-02T13:40:33Z" message: 'Readiness probe failed: dial tcp 10.134.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:39:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:40:33Z" name: error-404-isvc-e2bd2-predictor-677686d8bb-mxljv.18be7cafdf51c64f namespace: kserve-ci-e2e-test resourceVersion: "22254" uid: 95d2f54c-1cd6-4315-9417-b6c462111e3d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:49:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e2bd2-predictor-677686d8bb-mxljv namespace: kserve-ci-e2e-test resourceVersion: "21852" uid: e000058e-9278-448d-8993-fd666a913d9e kind: Event lastTimestamp: "2026-07-02T13:49:15Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T13:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:49:15Z" name: error-404-isvc-e2bd2-predictor-677686d8bb-mxljv.18be7d31d87e11c0 namespace: kserve-ci-e2e-test resourceVersion: "26542" uid: c1328702-5a65-4ed8-8e1b-defef53feecd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:49:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e2bd2-predictor-677686d8bb-mxljv namespace: kserve-ci-e2e-test resourceVersion: "21852" uid: e000058e-9278-448d-8993-fd666a913d9e kind: Event lastTimestamp: "2026-07-02T13:49:15Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:49:15Z" name: error-404-isvc-e2bd2-predictor-677686d8bb-mxljv.18be7d31d87e6984 namespace: kserve-ci-e2e-test resourceVersion: "26544" uid: 949b7600-d7fa-42df-a347-29fb4ef30e14 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:49:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e2bd2-predictor-677686d8bb-mxljv namespace: kserve-ci-e2e-test resourceVersion: "21852" uid: e000058e-9278-448d-8993-fd666a913d9e kind: Event lastTimestamp: "2026-07-02T13:49:18Z" message: 'Readiness probe failed: Get "https://10.134.0.39:8643/healthz": dial tcp 10.134.0.39:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:49:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:49:18Z" name: error-404-isvc-e2bd2-predictor-677686d8bb-mxljv.18be7d327dd7aeed namespace: kserve-ci-e2e-test resourceVersion: "26594" uid: b497f769-bcfc-453b-9c88-90dcf9a5b522 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:55Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-e2bd2-predictor-677686d8bb namespace: kserve-ci-e2e-test resourceVersion: "21841" uid: 2611baea-4111-47ab-b311-872a24e80f12 kind: Event lastTimestamp: "2026-07-02T13:39:55Z" message: 'Created pod: error-404-isvc-e2bd2-predictor-677686d8bb-mxljv' metadata: creationTimestamp: "2026-07-02T13:39:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:39:55Z" name: error-404-isvc-e2bd2-predictor-677686d8bb.18be7caf75d1d532 namespace: kserve-ci-e2e-test resourceVersion: "21851" uid: 67e31127-00d0-4038-8e0a-d3b54486df5e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:55Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-e2bd2-predictor namespace: kserve-ci-e2e-test resourceVersion: "21838" uid: 0a4e2fbc-135f-46fd-9b9b-73ee09fdbc50 kind: Event lastTimestamp: "2026-07-02T13:39:55Z" message: Scaled up replica set error-404-isvc-e2bd2-predictor-677686d8bb from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:39:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:39:55Z" name: error-404-isvc-e2bd2-predictor.18be7caf7440d5e6 namespace: kserve-ci-e2e-test resourceVersion: "21843" uid: 353c9161-6f2b-4871-9d9e-d474487ae6d9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:40:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e2bd2-predictor namespace: kserve-ci-e2e-test resourceVersion: "21840" uid: 87cd64f7-6cb4-4f10-94ef-d933e7b5b695 kind: Event lastTimestamp: "2026-07-02T13:40:25Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:40:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:40:25Z" name: error-404-isvc-e2bd2-predictor.18be7cb2f2a81287 namespace: kserve-ci-e2e-test resourceVersion: "22193" uid: 2405d7f3-12ba-4460-8621-4b3b563f41b8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:40:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e2bd2-predictor namespace: kserve-ci-e2e-test resourceVersion: "21840" uid: 87cd64f7-6cb4-4f10-94ef-d933e7b5b695 kind: Event lastTimestamp: "2026-07-02T13:40:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:40:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:40:25Z" name: error-404-isvc-e2bd2-predictor.18be7cb2f3087b87 namespace: kserve-ci-e2e-test resourceVersion: "22194" uid: e4457f78-3a3c-4420-9210-8fd94bac31e5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:40:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e2bd2-predictor namespace: kserve-ci-e2e-test resourceVersion: "22026" uid: 87cd64f7-6cb4-4f10-94ef-d933e7b5b695 kind: Event lastTimestamp: "2026-07-02T13:40:55Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:40:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:40:55Z" name: error-404-isvc-e2bd2-predictor.18be7cb9f0301408 namespace: kserve-ci-e2e-test resourceVersion: "22615" uid: f35c5078-97d6-4dd7-9bc4-6cfea4a2e2a9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:40:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e2bd2-predictor namespace: kserve-ci-e2e-test resourceVersion: "22026" uid: 87cd64f7-6cb4-4f10-94ef-d933e7b5b695 kind: Event lastTimestamp: "2026-07-02T13:40:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:40:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:40:55Z" name: error-404-isvc-e2bd2-predictor.18be7cb9f08a8238 namespace: kserve-ci-e2e-test resourceVersion: "22617" uid: e88aea6a-a528-4619-9ff2-5d8750cba434 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:39:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e2bd2 namespace: kserve-ci-e2e-test resourceVersion: "21824" uid: a5c38493-5ae2-4387-a6be-156de03f0a66 kind: Event lastTimestamp: "2026-07-02T13:40:43Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T13:39:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:40:43Z" name: error-404-isvc-e2bd2.18be7caf7444d534 namespace: kserve-ci-e2e-test resourceVersion: "22491" uid: cfda419d-1a80-47d7-ac1d-0e9eb3210d0b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e2bd2 namespace: kserve-ci-e2e-test resourceVersion: "21824" uid: a5c38493-5ae2-4387-a6be-156de03f0a66 kind: Event lastTimestamp: "2026-07-02T13:39:55Z" message: 'Failed to update status for InferenceService "error-404-isvc-e2bd2": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-e2bd2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:39:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:39:55Z" name: error-404-isvc-e2bd2.18be7caf748efcc6 namespace: kserve-ci-e2e-test resourceVersion: "21846" uid: 7fdf2a3e-e2f2-4fa1-af03-c2a1a55d5f84 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e2bd2 namespace: kserve-ci-e2e-test resourceVersion: "21824" uid: a5c38493-5ae2-4387-a6be-156de03f0a66 kind: Event lastTimestamp: "2026-07-02T13:39:55Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-e2bd2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:39:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:39:55Z" name: error-404-isvc-e2bd2.18be7caf748f69fe namespace: kserve-ci-e2e-test resourceVersion: "21847" uid: a0e724f0-be7d-4454-bae2-a6dea242bb81 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:40:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e2bd2 namespace: kserve-ci-e2e-test resourceVersion: "22488" uid: a5c38493-5ae2-4387-a6be-156de03f0a66 kind: Event lastTimestamp: "2026-07-02T13:40:43Z" message: InferenceService [error-404-isvc-e2bd2] is Ready metadata: creationTimestamp: "2026-07-02T13:40:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:40:43Z" name: error-404-isvc-e2bd2.18be7cba99f0992e namespace: kserve-ci-e2e-test resourceVersion: "22489" uid: ecdb9f1d-2fd0-46af-aa07-b20f9bdd34bf reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:30:10.164551Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f369f-predictor-784754f556-n6j5x namespace: kserve-ci-e2e-test resourceVersion: "17101" uid: 3ca3c6a8-3ce7-469a-bf23-76cea2611e36 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-f369f-predictor-784754f556-n6j5x to ip-10-0-133-122.ec2.internal metadata: creationTimestamp: "2026-07-02T13:30:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:30:10Z" name: error-404-isvc-f369f-predictor-784754f556-n6j5x.18be7c272bbb5881 namespace: kserve-ci-e2e-test resourceVersion: "17105" uid: 9409febf-7717-4c8e-8433-bbbc481d51de reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7f4bb8bc8-mnq2w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:30:10Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f369f-predictor-784754f556-n6j5x namespace: kserve-ci-e2e-test resourceVersion: "17112" uid: 3ca3c6a8-3ce7-469a-bf23-76cea2611e36 kind: Event lastTimestamp: "2026-07-02T13:30:10Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:30:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:30:10Z" name: error-404-isvc-f369f-predictor-784754f556-n6j5x.18be7c2744ec5cf3 namespace: kserve-ci-e2e-test resourceVersion: "17123" uid: 23f0cb3d-66b4-4c56-bcf9-636ad511a5b0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:30:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f369f-predictor-784754f556-n6j5x namespace: kserve-ci-e2e-test resourceVersion: "17103" uid: 3ca3c6a8-3ce7-469a-bf23-76cea2611e36 kind: Event lastTimestamp: "2026-07-02T13:30:10Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1683" already present on machine metadata: creationTimestamp: "2026-07-02T13:30:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:30:10Z" name: error-404-isvc-f369f-predictor-784754f556-n6j5x.18be7c2746389001 namespace: kserve-ci-e2e-test resourceVersion: "17125" uid: eae13819-11b3-418d-a845-ec2d9981c012 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:30:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f369f-predictor-784754f556-n6j5x namespace: kserve-ci-e2e-test resourceVersion: "17103" uid: 3ca3c6a8-3ce7-469a-bf23-76cea2611e36 kind: Event lastTimestamp: "2026-07-02T13:30:10Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T13:30:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:30:10Z" name: error-404-isvc-f369f-predictor-784754f556-n6j5x.18be7c27494bd277 namespace: kserve-ci-e2e-test resourceVersion: "17128" uid: 90c1b105-b4d8-441b-9f79-3810ea44bc55 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:30:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f369f-predictor-784754f556-n6j5x namespace: kserve-ci-e2e-test resourceVersion: "17103" uid: 3ca3c6a8-3ce7-469a-bf23-76cea2611e36 kind: Event lastTimestamp: "2026-07-02T13:30:10Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T13:30:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:30:10Z" name: error-404-isvc-f369f-predictor-784754f556-n6j5x.18be7c2749ae554c namespace: kserve-ci-e2e-test resourceVersion: "17129" uid: 23c76e64-d277-49da-a38b-5f9b597448f4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:30:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f369f-predictor-784754f556-n6j5x namespace: kserve-ci-e2e-test resourceVersion: "17103" uid: 3ca3c6a8-3ce7-469a-bf23-76cea2611e36 kind: Event lastTimestamp: "2026-07-02T13:30:10Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T13:30:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:30:10Z" name: error-404-isvc-f369f-predictor-784754f556-n6j5x.18be7c2749b93390 namespace: kserve-ci-e2e-test resourceVersion: "17130" uid: b81948c3-dddb-4aee-b6ad-e2d3b3c5b8e0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:30:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f369f-predictor-784754f556-n6j5x namespace: kserve-ci-e2e-test resourceVersion: "17103" uid: 3ca3c6a8-3ce7-469a-bf23-76cea2611e36 kind: Event lastTimestamp: "2026-07-02T13:30:10Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T13:30:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:30:10Z" name: error-404-isvc-f369f-predictor-784754f556-n6j5x.18be7c274c262af2 namespace: kserve-ci-e2e-test resourceVersion: "17132" uid: 5f0aec62-3c39-4a26-b84a-cb613dd71625 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:30:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f369f-predictor-784754f556-n6j5x namespace: kserve-ci-e2e-test resourceVersion: "17103" uid: 3ca3c6a8-3ce7-469a-bf23-76cea2611e36 kind: Event lastTimestamp: "2026-07-02T13:30:10Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:30:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:30:10Z" name: error-404-isvc-f369f-predictor-784754f556-n6j5x.18be7c274c878c63 namespace: kserve-ci-e2e-test resourceVersion: "17134" uid: be71e82e-1c7f-404b-9ed2-9d44c2169ab2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-02T13:30:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f369f-predictor-784754f556-n6j5x namespace: kserve-ci-e2e-test resourceVersion: "17103" uid: 3ca3c6a8-3ce7-469a-bf23-76cea2611e36 kind: Event lastTimestamp: "2026-07-02T13:39:26Z" message: 'Readiness probe failed: dial tcp 10.134.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:30:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:39:26Z" name: error-404-isvc-f369f-predictor-784754f556-n6j5x.18be7c2750323097 namespace: kserve-ci-e2e-test resourceVersion: "21454" uid: 52779e43-36ec-4569-a9a5-3c3b1036bef5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f369f-predictor-784754f556-n6j5x namespace: kserve-ci-e2e-test resourceVersion: "17103" uid: 3ca3c6a8-3ce7-469a-bf23-76cea2611e36 kind: Event lastTimestamp: "2026-07-02T13:39:24Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T13:39:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:39:24Z" name: error-404-isvc-f369f-predictor-784754f556-n6j5x.18be7ca851137381 namespace: kserve-ci-e2e-test resourceVersion: "21341" uid: 355c7bd2-5302-4992-a33a-4e36112cf261 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f369f-predictor-784754f556-n6j5x namespace: kserve-ci-e2e-test resourceVersion: "17103" uid: 3ca3c6a8-3ce7-469a-bf23-76cea2611e36 kind: Event lastTimestamp: "2026-07-02T13:39:24Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:39:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:39:24Z" name: error-404-isvc-f369f-predictor-784754f556-n6j5x.18be7ca851142f62 namespace: kserve-ci-e2e-test resourceVersion: "21344" uid: 33c31fbe-61e0-4e6a-84e9-7664a9248ba3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f369f-predictor-784754f556-n6j5x namespace: kserve-ci-e2e-test resourceVersion: "17103" uid: 3ca3c6a8-3ce7-469a-bf23-76cea2611e36 kind: Event lastTimestamp: "2026-07-02T13:39:26Z" message: 'Readiness probe failed: Get "https://10.134.0.33:8643/healthz": dial tcp 10.134.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:39:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:39:26Z" name: error-404-isvc-f369f-predictor-784754f556-n6j5x.18be7ca8c4995411 namespace: kserve-ci-e2e-test resourceVersion: "21453" uid: 73bdb8d2-1158-473b-b744-82f45809a522 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:30:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-f369f-predictor-784754f556 namespace: kserve-ci-e2e-test resourceVersion: "17095" uid: 47d92c18-6f43-4b56-a47d-d0410caa5a4e kind: Event lastTimestamp: "2026-07-02T13:30:10Z" message: 'Created pod: error-404-isvc-f369f-predictor-784754f556-n6j5x' metadata: creationTimestamp: "2026-07-02T13:30:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:30:10Z" name: error-404-isvc-f369f-predictor-784754f556.18be7c272b48eae7 namespace: kserve-ci-e2e-test resourceVersion: "17102" uid: d246a0a9-d7b5-4f69-8457-bc37633e416f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:30:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-f369f-predictor namespace: kserve-ci-e2e-test resourceVersion: "17092" uid: 0d1c92d6-aea4-4735-9e64-b18a89eff7b6 kind: Event lastTimestamp: "2026-07-02T13:30:10Z" message: Scaled up replica set error-404-isvc-f369f-predictor-784754f556 from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:30:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:30:10Z" name: error-404-isvc-f369f-predictor.18be7c2729c3c044 namespace: kserve-ci-e2e-test resourceVersion: "17096" uid: 3000dad5-84f2-4895-b079-fae633241488 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:30:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f369f-predictor namespace: kserve-ci-e2e-test resourceVersion: "17094" uid: 8b291be4-ffa1-4c60-affc-76c01b08c684 kind: Event lastTimestamp: "2026-07-02T13:30:40Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:30:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:30:40Z" name: error-404-isvc-f369f-predictor.18be7c2aa84e941b namespace: kserve-ci-e2e-test resourceVersion: "17458" uid: 1b2dee41-ea5b-4663-bc40-eb70e7df0761 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:30:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f369f-predictor namespace: kserve-ci-e2e-test resourceVersion: "17094" uid: 8b291be4-ffa1-4c60-affc-76c01b08c684 kind: Event lastTimestamp: "2026-07-02T13:30:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:30:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:30:40Z" name: error-404-isvc-f369f-predictor.18be7c2aa8aca7dd namespace: kserve-ci-e2e-test resourceVersion: "17459" uid: b0dbbac3-0ee2-4c4c-b717-60f0d45179db reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:30:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f369f-predictor namespace: kserve-ci-e2e-test resourceVersion: "17305" uid: 8b291be4-ffa1-4c60-affc-76c01b08c684 kind: Event lastTimestamp: "2026-07-02T13:31:10Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:30:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:31:10Z" name: error-404-isvc-f369f-predictor.18be7c31a5ad9ab9 namespace: kserve-ci-e2e-test resourceVersion: "17735" uid: af1571f6-cf26-4a71-8eff-1733f69e4297 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:30:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f369f-predictor namespace: kserve-ci-e2e-test resourceVersion: "17305" uid: 8b291be4-ffa1-4c60-affc-76c01b08c684 kind: Event lastTimestamp: "2026-07-02T13:31:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:30:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:31:10Z" name: error-404-isvc-f369f-predictor.18be7c31a60a5a5c namespace: kserve-ci-e2e-test resourceVersion: "17738" uid: 3a4d7b0e-082a-4e14-a3ea-63df83fde19a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:30:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f369f namespace: kserve-ci-e2e-test resourceVersion: "17060" uid: 3642a84f-fa6e-40a2-aff1-347a6b1d4a8b kind: Event lastTimestamp: "2026-07-02T13:30:56Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T13:30:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:30:56Z" name: error-404-isvc-f369f.18be7c2729e446c2 namespace: kserve-ci-e2e-test resourceVersion: "17599" uid: fb019f85-b857-4abb-9882-6c699043de04 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:30:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f369f namespace: kserve-ci-e2e-test resourceVersion: "17597" uid: 3642a84f-fa6e-40a2-aff1-347a6b1d4a8b kind: Event lastTimestamp: "2026-07-02T13:30:56Z" message: InferenceService [error-404-isvc-f369f] is Ready metadata: creationTimestamp: "2026-07-02T13:30:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:30:56Z" name: error-404-isvc-f369f.18be7c320ac3b5c1 namespace: kserve-ci-e2e-test resourceVersion: "17598" uid: 270a3452-ffbd-45e1-8f50-c2120e83033d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:27:40.056367Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-6f5c857d79-k8xjk namespace: kserve-ci-e2e-test resourceVersion: "15329" uid: e2f65642-bd7a-4bdf-90db-e1527c5f8a1b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-6f5c857d79-k8xjk to ip-10-0-131-238.ec2.internal metadata: creationTimestamp: "2026-07-02T13:27:40Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:27:40Z" name: isvc-sklearn-graph-1-predictor-6f5c857d79-k8xjk.18be7c0438963074 namespace: kserve-ci-e2e-test resourceVersion: "15334" uid: 155040b4-01c5-4a46-8f33-256757f57716 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7f4bb8bc8-mnq2w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:27:40Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-6f5c857d79-k8xjk namespace: kserve-ci-e2e-test resourceVersion: "15331" uid: e2f65642-bd7a-4bdf-90db-e1527c5f8a1b kind: Event lastTimestamp: "2026-07-02T13:27:40Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T13:27:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:27:40Z" name: isvc-sklearn-graph-1-predictor-6f5c857d79-k8xjk.18be7c04434a9548 namespace: kserve-ci-e2e-test resourceVersion: "15378" uid: 15f1da78-b95d-41bf-8f8f-850476cabffe reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:27:41Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-6f5c857d79-k8xjk namespace: kserve-ci-e2e-test resourceVersion: "15342" uid: e2f65642-bd7a-4bdf-90db-e1527c5f8a1b kind: Event lastTimestamp: "2026-07-02T13:27:41Z" message: Add eth0 [10.132.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:27:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:27:41Z" name: isvc-sklearn-graph-1-predictor-6f5c857d79-k8xjk.18be7c047f0386d2 namespace: kserve-ci-e2e-test resourceVersion: "15462" uid: 44dde73b-2416-480e-9531-5bf8bc1ddfb8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:27:41Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-6f5c857d79-k8xjk namespace: kserve-ci-e2e-test resourceVersion: "15331" uid: e2f65642-bd7a-4bdf-90db-e1527c5f8a1b kind: Event lastTimestamp: "2026-07-02T13:27:41Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:435cc5f190a526ac402884f9d2ad69617a79d5a6beac25c41b846f750a6a7ef1" already present on machine metadata: creationTimestamp: "2026-07-02T13:27:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:27:41Z" name: isvc-sklearn-graph-1-predictor-6f5c857d79-k8xjk.18be7c0480cfd793 namespace: kserve-ci-e2e-test resourceVersion: "15464" uid: dcbfe6ed-3917-41e9-aff1-b4561a30b64f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:27:41Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-6f5c857d79-k8xjk namespace: kserve-ci-e2e-test resourceVersion: "15331" uid: e2f65642-bd7a-4bdf-90db-e1527c5f8a1b kind: Event lastTimestamp: "2026-07-02T13:27:41Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T13:27:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:27:41Z" name: isvc-sklearn-graph-1-predictor-6f5c857d79-k8xjk.18be7c0485ff9332 namespace: kserve-ci-e2e-test resourceVersion: "15465" uid: 6b04c88b-6233-4476-809d-144f3da97aeb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:27:41Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-6f5c857d79-k8xjk namespace: kserve-ci-e2e-test resourceVersion: "15331" uid: e2f65642-bd7a-4bdf-90db-e1527c5f8a1b kind: Event lastTimestamp: "2026-07-02T13:27:41Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T13:27:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:27:41Z" name: isvc-sklearn-graph-1-predictor-6f5c857d79-k8xjk.18be7c0487317056 namespace: kserve-ci-e2e-test resourceVersion: "15466" uid: 3fc3b3b7-f2f7-41f8-a385-43f7536dda10 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:27:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6f5c857d79-k8xjk namespace: kserve-ci-e2e-test resourceVersion: "15331" uid: e2f65642-bd7a-4bdf-90db-e1527c5f8a1b kind: Event lastTimestamp: "2026-07-02T13:27:46Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1683" metadata: creationTimestamp: "2026-07-02T13:27:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:27:46Z" name: isvc-sklearn-graph-1-predictor-6f5c857d79-k8xjk.18be7c05a10d2715 namespace: kserve-ci-e2e-test resourceVersion: "15518" uid: e309ce80-a249-4993-93ae-31ee6bf03b49 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:28:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6f5c857d79-k8xjk namespace: kserve-ci-e2e-test resourceVersion: "15331" uid: e2f65642-bd7a-4bdf-90db-e1527c5f8a1b kind: Event lastTimestamp: "2026-07-02T13:28:02Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1683" in 16.389s (16.389s including waiting). Image size: 1558595490 bytes.' metadata: creationTimestamp: "2026-07-02T13:28:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:28:02Z" name: isvc-sklearn-graph-1-predictor-6f5c857d79-k8xjk.18be7c0971f5e480 namespace: kserve-ci-e2e-test resourceVersion: "15670" uid: 45ebfdd8-a82d-40a9-88a2-fa52f2429e50 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:28:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6f5c857d79-k8xjk namespace: kserve-ci-e2e-test resourceVersion: "15331" uid: e2f65642-bd7a-4bdf-90db-e1527c5f8a1b kind: Event lastTimestamp: "2026-07-02T13:28:02Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T13:28:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:28:02Z" name: isvc-sklearn-graph-1-predictor-6f5c857d79-k8xjk.18be7c097603d65a namespace: kserve-ci-e2e-test resourceVersion: "15676" uid: 38db5fc8-329e-47d4-bfc1-be970aba5f63 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:28:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6f5c857d79-k8xjk namespace: kserve-ci-e2e-test resourceVersion: "15331" uid: e2f65642-bd7a-4bdf-90db-e1527c5f8a1b kind: Event lastTimestamp: "2026-07-02T13:28:02Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T13:28:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:28:02Z" name: isvc-sklearn-graph-1-predictor-6f5c857d79-k8xjk.18be7c0976f81b71 namespace: kserve-ci-e2e-test resourceVersion: "15680" uid: 5c225997-04ad-4e88-b947-20650c45de9b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:28:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6f5c857d79-k8xjk namespace: kserve-ci-e2e-test resourceVersion: "15331" uid: e2f65642-bd7a-4bdf-90db-e1527c5f8a1b kind: Event lastTimestamp: "2026-07-02T13:28:02Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-02T13:28:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:28:02Z" name: isvc-sklearn-graph-1-predictor-6f5c857d79-k8xjk.18be7c0977083c66 namespace: kserve-ci-e2e-test resourceVersion: "15681" uid: 64c7f962-7eb2-4a93-8467-0c0e54786260 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:28:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6f5c857d79-k8xjk namespace: kserve-ci-e2e-test resourceVersion: "15331" uid: e2f65642-bd7a-4bdf-90db-e1527c5f8a1b kind: Event lastTimestamp: "2026-07-02T13:28:06Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 4.054s (4.054s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-02T13:28:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:28:06Z" name: isvc-sklearn-graph-1-predictor-6f5c857d79-k8xjk.18be7c0a68adaf4b namespace: kserve-ci-e2e-test resourceVersion: "15710" uid: 1335deab-f4e9-4aa4-b05a-7077d4c74b62 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:28:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6f5c857d79-k8xjk namespace: kserve-ci-e2e-test resourceVersion: "15331" uid: e2f65642-bd7a-4bdf-90db-e1527c5f8a1b kind: Event lastTimestamp: "2026-07-02T13:28:06Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T13:28:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:28:06Z" name: isvc-sklearn-graph-1-predictor-6f5c857d79-k8xjk.18be7c0a6da638eb namespace: kserve-ci-e2e-test resourceVersion: "15713" uid: 158d9a6a-8add-4c58-b30c-a7ee6d855867 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:28:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6f5c857d79-k8xjk namespace: kserve-ci-e2e-test resourceVersion: "15331" uid: e2f65642-bd7a-4bdf-90db-e1527c5f8a1b kind: Event lastTimestamp: "2026-07-02T13:28:06Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:28:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:28:06Z" name: isvc-sklearn-graph-1-predictor-6f5c857d79-k8xjk.18be7c0a6e35237c namespace: kserve-ci-e2e-test resourceVersion: "15714" uid: 2592fece-2444-4acb-954b-ea14b6d3fa6b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-02T13:28:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6f5c857d79-k8xjk namespace: kserve-ci-e2e-test resourceVersion: "15331" uid: e2f65642-bd7a-4bdf-90db-e1527c5f8a1b kind: Event lastTimestamp: "2026-07-02T13:29:24Z" message: 'Readiness probe failed: dial tcp 10.132.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:28:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:29:24Z" name: isvc-sklearn-graph-1-predictor-6f5c857d79-k8xjk.18be7c0acb717df8 namespace: kserve-ci-e2e-test resourceVersion: "16417" uid: fa38acb0-2590-4693-a5b4-b8453519839b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:30:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6f5c857d79-k8xjk namespace: kserve-ci-e2e-test resourceVersion: "15331" uid: e2f65642-bd7a-4bdf-90db-e1527c5f8a1b kind: Event lastTimestamp: "2026-07-02T13:30:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T13:30:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:30:10Z" name: isvc-sklearn-graph-1-predictor-6f5c857d79-k8xjk.18be7c27288d538a namespace: kserve-ci-e2e-test resourceVersion: "17088" uid: a6593c1b-de0c-47e4-93bd-abb9098dd948 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:30:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6f5c857d79-k8xjk namespace: kserve-ci-e2e-test resourceVersion: "15331" uid: e2f65642-bd7a-4bdf-90db-e1527c5f8a1b kind: Event lastTimestamp: "2026-07-02T13:30:10Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:30:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:30:10Z" name: isvc-sklearn-graph-1-predictor-6f5c857d79-k8xjk.18be7c27288e2db8 namespace: kserve-ci-e2e-test resourceVersion: "17091" uid: 289cf25f-b182-4721-bae5-0cd433d8fa3a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:30:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6f5c857d79-k8xjk namespace: kserve-ci-e2e-test resourceVersion: "15331" uid: e2f65642-bd7a-4bdf-90db-e1527c5f8a1b kind: Event lastTimestamp: "2026-07-02T13:30:14Z" message: 'Readiness probe failed: Get "https://10.132.0.32:8643/healthz": dial tcp 10.132.0.32:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:30:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:30:14Z" name: isvc-sklearn-graph-1-predictor-6f5c857d79-k8xjk.18be7c2821eef283 namespace: kserve-ci-e2e-test resourceVersion: "17183" uid: 03601777-34c3-47d6-a761-105fd79b27e6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:27:40Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-6f5c857d79 namespace: kserve-ci-e2e-test resourceVersion: "15318" uid: bc3e626d-4249-4687-b281-23c6da7f8c3c kind: Event lastTimestamp: "2026-07-02T13:27:40Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-6f5c857d79-k8xjk' metadata: creationTimestamp: "2026-07-02T13:27:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:27:40Z" name: isvc-sklearn-graph-1-predictor-6f5c857d79.18be7c0438372a56 namespace: kserve-ci-e2e-test resourceVersion: "15330" uid: dacf7c04-0f0e-4689-8620-98e910ba77a6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:27:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15316" uid: a63bef5d-85e4-43a1-979a-317c0ab174d9 kind: Event lastTimestamp: "2026-07-02T13:27:39Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-6f5c857d79 from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:27:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:27:39Z" name: isvc-sklearn-graph-1-predictor.18be7c043450ae74 namespace: kserve-ci-e2e-test resourceVersion: "15319" uid: 206a8c63-f113-45db-b27a-08f42607970f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:27:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15317" uid: bde9a8f9-da37-4a01-a7b9-6fe34137e407 kind: Event lastTimestamp: "2026-07-02T13:28:40Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:27:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:28:40Z" name: isvc-sklearn-graph-1-predictor.18be7c07b3509b3c namespace: kserve-ci-e2e-test resourceVersion: "16012" uid: 600444fc-18b8-41dd-a361-f6551159e079 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:27:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15317" uid: bde9a8f9-da37-4a01-a7b9-6fe34137e407 kind: Event lastTimestamp: "2026-07-02T13:28:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:27:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:28:40Z" name: isvc-sklearn-graph-1-predictor.18be7c07b3ab3ee6 namespace: kserve-ci-e2e-test resourceVersion: "16013" uid: 7773c3be-2a42-4265-9b10-342c0b5c5dac reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T13:28:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15593" uid: bde9a8f9-da37-4a01-a7b9-6fe34137e407 kind: Event lastTimestamp: "2026-07-02T13:29:40Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:28:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:29:40Z" name: isvc-sklearn-graph-1-predictor.18be7c0eb1332723 namespace: kserve-ci-e2e-test resourceVersion: "16716" uid: 405c6c88-ea11-4396-85f1-b67d4ac302bb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T13:28:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15593" uid: bde9a8f9-da37-4a01-a7b9-6fe34137e407 kind: Event lastTimestamp: "2026-07-02T13:29:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:28:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:29:40Z" name: isvc-sklearn-graph-1-predictor.18be7c0eb191a013 namespace: kserve-ci-e2e-test resourceVersion: "16717" uid: ef8bdb31-524e-4079-a2de-b7c1eba2911a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-02T13:27:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "15302" uid: 16ddf83e-6407-4c41-b419-f28ea1e48442 kind: Event lastTimestamp: "2026-07-02T13:29:29Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T13:27:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:29:29Z" name: isvc-sklearn-graph-1.18be7c0434506ad8 namespace: kserve-ci-e2e-test resourceVersion: "16497" uid: d059718e-e070-4826-b6e3-587e46f8b698 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:29:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "16494" uid: 16ddf83e-6407-4c41-b419-f28ea1e48442 kind: Event lastTimestamp: "2026-07-02T13:29:29Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-07-02T13:29:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:29:29Z" name: isvc-sklearn-graph-1.18be7c1dcfd0b319 namespace: kserve-ci-e2e-test resourceVersion: "16495" uid: 7608d22f-f947-457e-ab3a-3abfc283a696 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:27:40.358970Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-6fb9776d89-b7dzn namespace: kserve-ci-e2e-test resourceVersion: "15408" uid: 0528140d-dab1-4da1-8248-e6e511549ddf kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-6fb9776d89-b7dzn to ip-10-0-131-238.ec2.internal metadata: creationTimestamp: "2026-07-02T13:27:40Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:27:40Z" name: isvc-sklearn-graph-2-predictor-6fb9776d89-b7dzn.18be7c044a9f876d namespace: kserve-ci-e2e-test resourceVersion: "15412" uid: 99ee9536-da07-4cf2-bc62-2c13f6a3c3f0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7f4bb8bc8-mnq2w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:27:40Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-6fb9776d89-b7dzn namespace: kserve-ci-e2e-test resourceVersion: "15410" uid: 0528140d-dab1-4da1-8248-e6e511549ddf kind: Event lastTimestamp: "2026-07-02T13:27:40Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T13:27:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:27:40Z" name: isvc-sklearn-graph-2-predictor-6fb9776d89-b7dzn.18be7c0455735073 namespace: kserve-ci-e2e-test resourceVersion: "15434" uid: 4d805604-a3df-43b4-9e56-49f661028927 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:27:41Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-6fb9776d89-b7dzn namespace: kserve-ci-e2e-test resourceVersion: "15419" uid: 0528140d-dab1-4da1-8248-e6e511549ddf kind: Event lastTimestamp: "2026-07-02T13:27:41Z" message: Add eth0 [10.132.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:27:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:27:41Z" name: isvc-sklearn-graph-2-predictor-6fb9776d89-b7dzn.18be7c048d9c9546 namespace: kserve-ci-e2e-test resourceVersion: "15468" uid: 4e85938e-6432-4d8e-94c3-c74880b9dbfc reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:27:41Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-6fb9776d89-b7dzn namespace: kserve-ci-e2e-test resourceVersion: "15410" uid: 0528140d-dab1-4da1-8248-e6e511549ddf kind: Event lastTimestamp: "2026-07-02T13:27:41Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:435cc5f190a526ac402884f9d2ad69617a79d5a6beac25c41b846f750a6a7ef1" already present on machine metadata: creationTimestamp: "2026-07-02T13:27:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:27:41Z" name: isvc-sklearn-graph-2-predictor-6fb9776d89-b7dzn.18be7c048f90f946 namespace: kserve-ci-e2e-test resourceVersion: "15470" uid: 8e2c8895-cf54-4bbe-8bb9-b4d7b27c7a0d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:27:41Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-6fb9776d89-b7dzn namespace: kserve-ci-e2e-test resourceVersion: "15410" uid: 0528140d-dab1-4da1-8248-e6e511549ddf kind: Event lastTimestamp: "2026-07-02T13:27:41Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T13:27:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:27:41Z" name: isvc-sklearn-graph-2-predictor-6fb9776d89-b7dzn.18be7c04941cf2f9 namespace: kserve-ci-e2e-test resourceVersion: "15472" uid: daf425e3-9cf0-4b46-99ac-55e783c2689e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:27:41Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-6fb9776d89-b7dzn namespace: kserve-ci-e2e-test resourceVersion: "15410" uid: 0528140d-dab1-4da1-8248-e6e511549ddf kind: Event lastTimestamp: "2026-07-02T13:27:41Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T13:27:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:27:41Z" name: isvc-sklearn-graph-2-predictor-6fb9776d89-b7dzn.18be7c0494f6a00a namespace: kserve-ci-e2e-test resourceVersion: "15473" uid: 81b1565c-6d91-4189-99db-bf461c4a6c15 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:27:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6fb9776d89-b7dzn namespace: kserve-ci-e2e-test resourceVersion: "15410" uid: 0528140d-dab1-4da1-8248-e6e511549ddf kind: Event lastTimestamp: "2026-07-02T13:27:46Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1683" metadata: creationTimestamp: "2026-07-02T13:27:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:27:46Z" name: isvc-sklearn-graph-2-predictor-6fb9776d89-b7dzn.18be7c05a0f4bb8f namespace: kserve-ci-e2e-test resourceVersion: "15516" uid: d12dadee-a248-4946-bc4e-67c337b5bb59 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:28:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6fb9776d89-b7dzn namespace: kserve-ci-e2e-test resourceVersion: "15410" uid: 0528140d-dab1-4da1-8248-e6e511549ddf kind: Event lastTimestamp: "2026-07-02T13:28:02Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1683" in 16.374s (16.374s including waiting). Image size: 1558595490 bytes.' metadata: creationTimestamp: "2026-07-02T13:28:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:28:02Z" name: isvc-sklearn-graph-2-predictor-6fb9776d89-b7dzn.18be7c0970f7322d namespace: kserve-ci-e2e-test resourceVersion: "15669" uid: 8f2093cc-e962-4224-97f3-19bc1e63489f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:28:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6fb9776d89-b7dzn namespace: kserve-ci-e2e-test resourceVersion: "15410" uid: 0528140d-dab1-4da1-8248-e6e511549ddf kind: Event lastTimestamp: "2026-07-02T13:28:02Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T13:28:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:28:02Z" name: isvc-sklearn-graph-2-predictor-6fb9776d89-b7dzn.18be7c0975afca7b namespace: kserve-ci-e2e-test resourceVersion: "15675" uid: 167e05d6-4935-4242-8daa-d3934851cbfa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:28:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6fb9776d89-b7dzn namespace: kserve-ci-e2e-test resourceVersion: "15410" uid: 0528140d-dab1-4da1-8248-e6e511549ddf kind: Event lastTimestamp: "2026-07-02T13:28:02Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T13:28:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:28:02Z" name: isvc-sklearn-graph-2-predictor-6fb9776d89-b7dzn.18be7c09765ee7cb namespace: kserve-ci-e2e-test resourceVersion: "15677" uid: 43bb2b10-4673-4f8d-b450-7904eb5ccc10 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:28:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6fb9776d89-b7dzn namespace: kserve-ci-e2e-test resourceVersion: "15410" uid: 0528140d-dab1-4da1-8248-e6e511549ddf kind: Event lastTimestamp: "2026-07-02T13:28:02Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-02T13:28:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:28:02Z" name: isvc-sklearn-graph-2-predictor-6fb9776d89-b7dzn.18be7c097671e3ea namespace: kserve-ci-e2e-test resourceVersion: "15679" uid: 2e0ef745-ae05-48db-ad06-2b94c7c7c613 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:28:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6fb9776d89-b7dzn namespace: kserve-ci-e2e-test resourceVersion: "15410" uid: 0528140d-dab1-4da1-8248-e6e511549ddf kind: Event lastTimestamp: "2026-07-02T13:28:06Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 4.077s (4.077s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-02T13:28:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:28:06Z" name: isvc-sklearn-graph-2-predictor-6fb9776d89-b7dzn.18be7c0a6977c6f5 namespace: kserve-ci-e2e-test resourceVersion: "15711" uid: 89e6cda5-a409-4e49-8b06-a9a1c9cf9862 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:28:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6fb9776d89-b7dzn namespace: kserve-ci-e2e-test resourceVersion: "15410" uid: 0528140d-dab1-4da1-8248-e6e511549ddf kind: Event lastTimestamp: "2026-07-02T13:28:06Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T13:28:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:28:06Z" name: isvc-sklearn-graph-2-predictor-6fb9776d89-b7dzn.18be7c0a6ec9ec8f namespace: kserve-ci-e2e-test resourceVersion: "15715" uid: eabb3981-7a24-4037-9664-45af3e1168cc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:28:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6fb9776d89-b7dzn namespace: kserve-ci-e2e-test resourceVersion: "15410" uid: 0528140d-dab1-4da1-8248-e6e511549ddf kind: Event lastTimestamp: "2026-07-02T13:28:06Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:28:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:28:06Z" name: isvc-sklearn-graph-2-predictor-6fb9776d89-b7dzn.18be7c0a6faf5570 namespace: kserve-ci-e2e-test resourceVersion: "15719" uid: b1234715-a2cc-40a4-9709-0f5fe606255d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-02T13:28:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6fb9776d89-b7dzn namespace: kserve-ci-e2e-test resourceVersion: "15410" uid: 0528140d-dab1-4da1-8248-e6e511549ddf kind: Event lastTimestamp: "2026-07-02T13:29:17Z" message: 'Readiness probe failed: dial tcp 10.132.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:28:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:29:17Z" name: isvc-sklearn-graph-2-predictor-6fb9776d89-b7dzn.18be7c0acb70381b namespace: kserve-ci-e2e-test resourceVersion: "16326" uid: edcbf2cc-5b5f-45a1-a6a5-d81b1f905d8f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:30:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6fb9776d89-b7dzn namespace: kserve-ci-e2e-test resourceVersion: "15410" uid: 0528140d-dab1-4da1-8248-e6e511549ddf kind: Event lastTimestamp: "2026-07-02T13:30:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T13:30:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:30:10Z" name: isvc-sklearn-graph-2-predictor-6fb9776d89-b7dzn.18be7c27252cfced namespace: kserve-ci-e2e-test resourceVersion: "17054" uid: d4b74afc-47a0-450f-9e35-0924b8396da6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:30:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6fb9776d89-b7dzn namespace: kserve-ci-e2e-test resourceVersion: "15410" uid: 0528140d-dab1-4da1-8248-e6e511549ddf kind: Event lastTimestamp: "2026-07-02T13:30:10Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:30:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:30:10Z" name: isvc-sklearn-graph-2-predictor-6fb9776d89-b7dzn.18be7c27252e19b1 namespace: kserve-ci-e2e-test resourceVersion: "17055" uid: 1ceeb7dd-5f3f-4c85-9402-4f1212004d0c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:30:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6fb9776d89-b7dzn namespace: kserve-ci-e2e-test resourceVersion: "15410" uid: 0528140d-dab1-4da1-8248-e6e511549ddf kind: Event lastTimestamp: "2026-07-02T13:30:14Z" message: 'Readiness probe failed: Get "https://10.132.0.34:8643/healthz": dial tcp 10.132.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:30:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:30:14Z" name: isvc-sklearn-graph-2-predictor-6fb9776d89-b7dzn.18be7c2821edbca0 namespace: kserve-ci-e2e-test resourceVersion: "17182" uid: 96619fba-a6fb-41f5-bc8f-ccc45ff9530e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:27:40Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-6fb9776d89 namespace: kserve-ci-e2e-test resourceVersion: "15398" uid: 9b34f00b-984e-489b-8b37-596290e7a8ba kind: Event lastTimestamp: "2026-07-02T13:27:40Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-6fb9776d89-b7dzn' metadata: creationTimestamp: "2026-07-02T13:27:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:27:40Z" name: isvc-sklearn-graph-2-predictor-6fb9776d89.18be7c044a1787b5 namespace: kserve-ci-e2e-test resourceVersion: "15409" uid: 03c960d0-5600-4d92-a68f-cb828fd0409c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:27:40Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15395" uid: 87e63579-a35c-43e6-8a15-9e6c258735fa kind: Event lastTimestamp: "2026-07-02T13:27:40Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-6fb9776d89 from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:27:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:27:40Z" name: isvc-sklearn-graph-2-predictor.18be7c04471ee531 namespace: kserve-ci-e2e-test resourceVersion: "15400" uid: 69ac3f70-04ad-455a-a374-ae28c0718386 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:27:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15397" uid: ca490a1f-63d8-448a-8694-5671d6a6eb74 kind: Event lastTimestamp: "2026-07-02T13:28:10Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:27:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:28:10Z" name: isvc-sklearn-graph-2-predictor.18be7c07c56ebb43 namespace: kserve-ci-e2e-test resourceVersion: "15761" uid: fdf09ba5-93d8-4443-b787-6d3cc9b05596 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:27:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15397" uid: ca490a1f-63d8-448a-8694-5671d6a6eb74 kind: Event lastTimestamp: "2026-07-02T13:28:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:27:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:28:10Z" name: isvc-sklearn-graph-2-predictor.18be7c07c5cb431d namespace: kserve-ci-e2e-test resourceVersion: "15762" uid: 2c2663bf-e8ea-4a70-99df-86a63f94da63 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-02T13:28:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15609" uid: ca490a1f-63d8-448a-8694-5671d6a6eb74 kind: Event lastTimestamp: "2026-07-02T13:29:55Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:28:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:29:55Z" name: isvc-sklearn-graph-2-predictor.18be7c0ec2f3f0fa namespace: kserve-ci-e2e-test resourceVersion: "16843" uid: 9ceaad86-62dd-48d4-8f29-ca2b189f7e6d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-02T13:28:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15609" uid: ca490a1f-63d8-448a-8694-5671d6a6eb74 kind: Event lastTimestamp: "2026-07-02T13:29:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:28:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:29:55Z" name: isvc-sklearn-graph-2-predictor.18be7c0ec352b634 namespace: kserve-ci-e2e-test resourceVersion: "16844" uid: fe5b91ad-08ad-4a70-9691-810a31d88f7c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-02T13:27:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "15377" uid: 05c2b85a-024f-446c-8c93-c7d9b4d1b3f9 kind: Event lastTimestamp: "2026-07-02T13:29:27Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T13:27:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:29:27Z" name: isvc-sklearn-graph-2.18be7c04472246e3 namespace: kserve-ci-e2e-test resourceVersion: "16472" uid: 1091bcd1-e064-44a0-bc23-0c6f015c57eb reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:29:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "16468" uid: 05c2b85a-024f-446c-8c93-c7d9b4d1b3f9 kind: Event lastTimestamp: "2026-07-02T13:29:27Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-07-02T13:29:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:29:27Z" name: isvc-sklearn-graph-2.18be7c1d58ab61fa namespace: kserve-ci-e2e-test resourceVersion: "16469" uid: 053402d1-20af-4b07-b787-b7243f6fc430 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:27:40.256966Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-p6mls namespace: kserve-ci-e2e-test resourceVersion: "15379" uid: aed4e2c7-8093-4b8d-bbc4-f30bbc0f20d3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-555fc69d56-p6mls to ip-10-0-131-238.ec2.internal metadata: creationTimestamp: "2026-07-02T13:27:40Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:27:40Z" name: isvc-xgboost-graph-predictor-555fc69d56-p6mls.18be7c04448b1321 namespace: kserve-ci-e2e-test resourceVersion: "15384" uid: 302c05b2-5bc5-4b22-8b03-b8a0db5ac822 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7f4bb8bc8-mnq2w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:27:40Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-p6mls namespace: kserve-ci-e2e-test resourceVersion: "15392" uid: aed4e2c7-8093-4b8d-bbc4-f30bbc0f20d3 kind: Event lastTimestamp: "2026-07-02T13:27:40Z" message: Add eth0 [10.132.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:27:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:27:40Z" name: isvc-xgboost-graph-predictor-555fc69d56-p6mls.18be7c045f7b7523 namespace: kserve-ci-e2e-test resourceVersion: "15444" uid: 2548a9dc-4e91-47f9-b7e0-817abdc3f76e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:27:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-p6mls namespace: kserve-ci-e2e-test resourceVersion: "15381" uid: aed4e2c7-8093-4b8d-bbc4-f30bbc0f20d3 kind: Event lastTimestamp: "2026-07-02T13:27:40Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:435cc5f190a526ac402884f9d2ad69617a79d5a6beac25c41b846f750a6a7ef1" already present on machine metadata: creationTimestamp: "2026-07-02T13:27:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:27:40Z" name: isvc-xgboost-graph-predictor-555fc69d56-p6mls.18be7c0460bfe20b namespace: kserve-ci-e2e-test resourceVersion: "15446" uid: 278feec7-ffc7-49a5-acf4-63bdd77a31fc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:27:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-p6mls namespace: kserve-ci-e2e-test resourceVersion: "15381" uid: aed4e2c7-8093-4b8d-bbc4-f30bbc0f20d3 kind: Event lastTimestamp: "2026-07-02T13:27:40Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T13:27:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:27:40Z" name: isvc-xgboost-graph-predictor-555fc69d56-p6mls.18be7c04639c6ba0 namespace: kserve-ci-e2e-test resourceVersion: "15448" uid: 121bb75d-c73d-4083-b537-ac00b31e2a60 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:27:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-p6mls namespace: kserve-ci-e2e-test resourceVersion: "15381" uid: aed4e2c7-8093-4b8d-bbc4-f30bbc0f20d3 kind: Event lastTimestamp: "2026-07-02T13:27:40Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T13:27:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:27:40Z" name: isvc-xgboost-graph-predictor-555fc69d56-p6mls.18be7c0464166911 namespace: kserve-ci-e2e-test resourceVersion: "15450" uid: db14c814-7359-4c80-9fb9-260429100e4b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:27:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-p6mls namespace: kserve-ci-e2e-test resourceVersion: "15381" uid: aed4e2c7-8093-4b8d-bbc4-f30bbc0f20d3 kind: Event lastTimestamp: "2026-07-02T13:27:46Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-02T13:27:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:27:46Z" name: isvc-xgboost-graph-predictor-555fc69d56-p6mls.18be7c05a0def263 namespace: kserve-ci-e2e-test resourceVersion: "15515" uid: bf7246d3-7765-4ad2-9d1d-e100c9fad05a reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:28:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-p6mls namespace: kserve-ci-e2e-test resourceVersion: "15381" uid: aed4e2c7-8093-4b8d-bbc4-f30bbc0f20d3 kind: Event lastTimestamp: "2026-07-02T13:28:16Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 30.535s (30.535s including waiting). Image size: 1305293048 bytes.' metadata: creationTimestamp: "2026-07-02T13:28:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:28:16Z" name: isvc-xgboost-graph-predictor-555fc69d56-p6mls.18be7c0cbcf3095a namespace: kserve-ci-e2e-test resourceVersion: "15817" uid: 266bc6b8-0b19-47b0-83ab-502b5e83f3d3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:28:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-p6mls namespace: kserve-ci-e2e-test resourceVersion: "15381" uid: aed4e2c7-8093-4b8d-bbc4-f30bbc0f20d3 kind: Event lastTimestamp: "2026-07-02T13:28:16Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T13:28:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:28:16Z" name: isvc-xgboost-graph-predictor-555fc69d56-p6mls.18be7c0cc05a7188 namespace: kserve-ci-e2e-test resourceVersion: "15818" uid: 8177d7de-493c-4b14-9a8f-25ee3c13e181 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:28:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-p6mls namespace: kserve-ci-e2e-test resourceVersion: "15381" uid: aed4e2c7-8093-4b8d-bbc4-f30bbc0f20d3 kind: Event lastTimestamp: "2026-07-02T13:28:16Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T13:28:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:28:16Z" name: isvc-xgboost-graph-predictor-555fc69d56-p6mls.18be7c0cc0c1e5ce namespace: kserve-ci-e2e-test resourceVersion: "15819" uid: 6ebdec55-0dec-43ac-b146-3bd750e60c95 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:28:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-p6mls namespace: kserve-ci-e2e-test resourceVersion: "15381" uid: aed4e2c7-8093-4b8d-bbc4-f30bbc0f20d3 kind: Event lastTimestamp: "2026-07-02T13:28:16Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T13:28:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:28:16Z" name: isvc-xgboost-graph-predictor-555fc69d56-p6mls.18be7c0cc0cd8c30 namespace: kserve-ci-e2e-test resourceVersion: "15820" uid: da08fcfc-8b92-4030-85f6-40c7c6a3252e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:28:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-p6mls namespace: kserve-ci-e2e-test resourceVersion: "15381" uid: aed4e2c7-8093-4b8d-bbc4-f30bbc0f20d3 kind: Event lastTimestamp: "2026-07-02T13:28:16Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T13:28:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:28:16Z" name: isvc-xgboost-graph-predictor-555fc69d56-p6mls.18be7c0cc46450c5 namespace: kserve-ci-e2e-test resourceVersion: "15823" uid: d3c42c07-f711-4045-b2e0-9bdcafa3948d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:28:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-p6mls namespace: kserve-ci-e2e-test resourceVersion: "15381" uid: aed4e2c7-8093-4b8d-bbc4-f30bbc0f20d3 kind: Event lastTimestamp: "2026-07-02T13:28:16Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:28:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:28:16Z" name: isvc-xgboost-graph-predictor-555fc69d56-p6mls.18be7c0cc4d1305b namespace: kserve-ci-e2e-test resourceVersion: "15824" uid: 46b4a195-1469-4b8b-9e92-9fb2d21ef1e2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-02T13:28:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-p6mls namespace: kserve-ci-e2e-test resourceVersion: "15381" uid: aed4e2c7-8093-4b8d-bbc4-f30bbc0f20d3 kind: Event lastTimestamp: "2026-07-02T13:30:14Z" message: 'Readiness probe failed: dial tcp 10.132.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:28:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:30:14Z" name: isvc-xgboost-graph-predictor-555fc69d56-p6mls.18be7c0d22ff3339 namespace: kserve-ci-e2e-test resourceVersion: "17186" uid: 3927ec35-4337-4b7d-ade7-2e4cb857d350 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:30:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-p6mls namespace: kserve-ci-e2e-test resourceVersion: "15381" uid: aed4e2c7-8093-4b8d-bbc4-f30bbc0f20d3 kind: Event lastTimestamp: "2026-07-02T13:30:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T13:30:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:30:10Z" name: isvc-xgboost-graph-predictor-555fc69d56-p6mls.18be7c27224d7670 namespace: kserve-ci-e2e-test resourceVersion: "17026" uid: 07f7864f-1da1-4766-931d-e7f836cb22e7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:30:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-p6mls namespace: kserve-ci-e2e-test resourceVersion: "15381" uid: aed4e2c7-8093-4b8d-bbc4-f30bbc0f20d3 kind: Event lastTimestamp: "2026-07-02T13:30:10Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:30:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:30:10Z" name: isvc-xgboost-graph-predictor-555fc69d56-p6mls.18be7c27224e4661 namespace: kserve-ci-e2e-test resourceVersion: "17028" uid: 3c0f4aba-82b8-4cb5-93fe-4623d3f460f3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:30:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-p6mls namespace: kserve-ci-e2e-test resourceVersion: "15381" uid: aed4e2c7-8093-4b8d-bbc4-f30bbc0f20d3 kind: Event lastTimestamp: "2026-07-02T13:30:14Z" message: 'Readiness probe failed: Get "https://10.132.0.33:8643/healthz": dial tcp 10.132.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:30:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:30:14Z" name: isvc-xgboost-graph-predictor-555fc69d56-p6mls.18be7c282565bf11 namespace: kserve-ci-e2e-test resourceVersion: "17185" uid: dd29d37b-05d5-4ba7-bd09-a8604e2bb29a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:27:40Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-555fc69d56 namespace: kserve-ci-e2e-test resourceVersion: "15369" uid: d022e391-4864-4530-a2a6-b34d4579609b kind: Event lastTimestamp: "2026-07-02T13:27:40Z" message: 'Created pod: isvc-xgboost-graph-predictor-555fc69d56-p6mls' metadata: creationTimestamp: "2026-07-02T13:27:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:27:40Z" name: isvc-xgboost-graph-predictor-555fc69d56.18be7c04440adcc7 namespace: kserve-ci-e2e-test resourceVersion: "15380" uid: 6477a5a6-4913-4968-aa56-ad6b7d81703f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:27:40Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15367" uid: 16348ac2-3724-45fa-9164-4a8155931d44 kind: Event lastTimestamp: "2026-07-02T13:27:40Z" message: Scaled up replica set isvc-xgboost-graph-predictor-555fc69d56 from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:27:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:27:40Z" name: isvc-xgboost-graph-predictor.18be7c04410a408a namespace: kserve-ci-e2e-test resourceVersion: "15370" uid: d066110b-f3a0-451f-a77c-76dba7d09844 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T13:27:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15368" uid: 3e45074c-e05f-40f4-ad39-429977abdcbe kind: Event lastTimestamp: "2026-07-02T13:28:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:27:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:28:55Z" name: isvc-xgboost-graph-predictor.18be7c07bf669c39 namespace: kserve-ci-e2e-test resourceVersion: "16147" uid: 4d50c987-a51b-495f-9335-c4ca723b82f5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T13:27:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15368" uid: 3e45074c-e05f-40f4-ad39-429977abdcbe kind: Event lastTimestamp: "2026-07-02T13:28:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:27:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:28:55Z" name: isvc-xgboost-graph-predictor.18be7c07bfbbcff2 namespace: kserve-ci-e2e-test resourceVersion: "16148" uid: 6f6cb730-95f9-4ae8-9c42-d6f78b9b6da4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:29:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15603" uid: 3e45074c-e05f-40f4-ad39-429977abdcbe kind: Event lastTimestamp: "2026-07-02T13:29:40Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:29:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:29:40Z" name: isvc-xgboost-graph-predictor.18be7c19388a1453 namespace: kserve-ci-e2e-test resourceVersion: "16718" uid: 2e8032b8-75bb-4e4b-8a29-56de76eb06e3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:29:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15603" uid: 3e45074c-e05f-40f4-ad39-429977abdcbe kind: Event lastTimestamp: "2026-07-02T13:29:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:29:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:29:40Z" name: isvc-xgboost-graph-predictor.18be7c1938e9ae4a namespace: kserve-ci-e2e-test resourceVersion: "16719" uid: 58c3f708-d227-431d-bfc3-cf09dee3bc5c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-02T13:27:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15352" uid: 1a7e2e2b-75ed-46fc-99d5-fa40a12d8157 kind: Event lastTimestamp: "2026-07-02T13:29:24Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T13:27:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:29:24Z" name: isvc-xgboost-graph.18be7c04410fa39d namespace: kserve-ci-e2e-test resourceVersion: "16428" uid: 126b9ea4-0179-4104-bf56-0515e9b59036 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:29:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "16426" uid: 1a7e2e2b-75ed-46fc-99d5-fa40a12d8157 kind: Event lastTimestamp: "2026-07-02T13:29:24Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-07-02T13:29:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:29:24Z" name: isvc-xgboost-graph.18be7c1c858bd536 namespace: kserve-ci-e2e-test resourceVersion: "16427" uid: 99e08197-bb10-459c-853e-333f0d107bfe reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:29:59.821078Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-9d455c95-bnm99 namespace: kserve-ci-e2e-test resourceVersion: "16887" uid: 2d7d32fc-a9c5-4896-a945-96ae0a7f4d5f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-9d455c95-bnm99 to ip-10-0-133-122.ec2.internal metadata: creationTimestamp: "2026-07-02T13:29:59Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:29:59Z" name: model-chainer-9d455c95-bnm99.18be7c24c33668fe namespace: kserve-ci-e2e-test resourceVersion: "16892" uid: b5f65596-2476-4dff-a860-2b5ea9ce45a9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7f4bb8bc8-mnq2w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:30:00Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-9d455c95-bnm99 namespace: kserve-ci-e2e-test resourceVersion: "16896" uid: 2d7d32fc-a9c5-4896-a945-96ae0a7f4d5f kind: Event lastTimestamp: "2026-07-02T13:30:00Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:30:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:30:00Z" name: model-chainer-9d455c95-bnm99.18be7c24dcf2b225 namespace: kserve-ci-e2e-test resourceVersion: "16910" uid: 49a2e3dd-93f8-432d-a09a-d2990556957c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:30:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-9d455c95-bnm99 namespace: kserve-ci-e2e-test resourceVersion: "16890" uid: 2d7d32fc-a9c5-4896-a945-96ae0a7f4d5f kind: Event lastTimestamp: "2026-07-02T13:30:00Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:9f703da9f2f5afd108b6816a57feadccff9213d4d8437a63e732fa3fbaa3fa9d" already present on machine metadata: creationTimestamp: "2026-07-02T13:30:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:30:00Z" name: model-chainer-9d455c95-bnm99.18be7c24de15b827 namespace: kserve-ci-e2e-test resourceVersion: "16913" uid: 5501b84c-2b53-4bc4-8712-58aa6e94acf8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:30:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-9d455c95-bnm99 namespace: kserve-ci-e2e-test resourceVersion: "16890" uid: 2d7d32fc-a9c5-4896-a945-96ae0a7f4d5f kind: Event lastTimestamp: "2026-07-02T13:30:00Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-07-02T13:30:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:30:00Z" name: model-chainer-9d455c95-bnm99.18be7c24e0fcedc7 namespace: kserve-ci-e2e-test resourceVersion: "16914" uid: 0d7f65d4-de08-4733-9a11-819d7a355b75 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:30:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-9d455c95-bnm99 namespace: kserve-ci-e2e-test resourceVersion: "16890" uid: 2d7d32fc-a9c5-4896-a945-96ae0a7f4d5f kind: Event lastTimestamp: "2026-07-02T13:30:00Z" message: Started container model-chainer metadata: creationTimestamp: "2026-07-02T13:30:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:30:00Z" name: model-chainer-9d455c95-bnm99.18be7c24e15fb294 namespace: kserve-ci-e2e-test resourceVersion: "16917" uid: 61bbb0c9-a5f9-4568-849f-6da85aaf9224 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:30:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-9d455c95-bnm99 namespace: kserve-ci-e2e-test resourceVersion: "16890" uid: 2d7d32fc-a9c5-4896-a945-96ae0a7f4d5f kind: Event lastTimestamp: "2026-07-02T13:30:09Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-07-02T13:30:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:30:09Z" name: model-chainer-9d455c95-bnm99.18be7c271d83b3e1 namespace: kserve-ci-e2e-test resourceVersion: "17004" uid: 6d13239a-f111-4283-8283-238b2f2a47a1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:30:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-9d455c95-bnm99 namespace: kserve-ci-e2e-test resourceVersion: "16890" uid: 2d7d32fc-a9c5-4896-a945-96ae0a7f4d5f kind: Event lastTimestamp: "2026-07-02T13:30:36Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T13:30:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:30:36Z" name: model-chainer-9d455c95-bnm99.18be7c27899638cd namespace: kserve-ci-e2e-test resourceVersion: "17425" uid: f4c3a7c4-c1ae-4dae-847c-bee04f297172 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:29:59Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-9d455c95 namespace: kserve-ci-e2e-test resourceVersion: "16883" uid: 67b87dcf-5886-4757-9131-7d1e41da6eea kind: Event lastTimestamp: "2026-07-02T13:29:59Z" message: 'Created pod: model-chainer-9d455c95-bnm99' metadata: creationTimestamp: "2026-07-02T13:29:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:29:59Z" name: model-chainer-9d455c95.18be7c24c2df14ec namespace: kserve-ci-e2e-test resourceVersion: "16889" uid: bd0e68e7-5037-4e82-b14c-91cf03533f3c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:29:59Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16881" uid: 516fbb52-cfa7-43a8-a89b-18af65b4e8b8 kind: Event lastTimestamp: "2026-07-02T13:29:59Z" message: Scaled up replica set model-chainer-9d455c95 from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:29:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:29:59Z" name: model-chainer.18be7c24c1fc37b2 namespace: kserve-ci-e2e-test resourceVersion: "16885" uid: 93fecc88-63c3-47af-acc2-976d6e7cb264 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:29:59Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16898" uid: f0de0c23-f3c4-4096-be17-5f2d09965fb8 kind: Event lastTimestamp: "2026-07-02T13:29:59Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-07-02T13:29:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:29:59Z" name: model-chainer.18be7c24c6373330 namespace: kserve-ci-e2e-test resourceVersion: "16901" uid: 6c0313ed-54bd-4f0e-b113-5bc7b235c69a reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:40:24.914021Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-2bc9c-6db49dd9b7-dfmqj namespace: kserve-ci-e2e-test resourceVersion: "22162" uid: 8dc5fffe-c874-4979-a16b-307f7b3e2f53 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-2bc9c-6db49dd9b7-dfmqj to ip-10-0-133-122.ec2.internal metadata: creationTimestamp: "2026-07-02T13:40:24Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:40:24Z" name: sequence-graph-2bc9c-6db49dd9b7-dfmqj.18be7cb64da7d7db namespace: kserve-ci-e2e-test resourceVersion: "22167" uid: b45d067c-ed3d-4a12-8cd2-f83a16bde380 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7f4bb8bc8-mnq2w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:40:25Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-2bc9c-6db49dd9b7-dfmqj namespace: kserve-ci-e2e-test resourceVersion: "22173" uid: 8dc5fffe-c874-4979-a16b-307f7b3e2f53 kind: Event lastTimestamp: "2026-07-02T13:40:25Z" message: Add eth0 [10.134.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:40:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:40:25Z" name: sequence-graph-2bc9c-6db49dd9b7-dfmqj.18be7cb6677b52c4 namespace: kserve-ci-e2e-test resourceVersion: "22186" uid: d27d1b27-35fc-477c-b5f6-bfb6a00ca279 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:40:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-2bc9c} kind: Pod name: sequence-graph-2bc9c-6db49dd9b7-dfmqj namespace: kserve-ci-e2e-test resourceVersion: "22164" uid: 8dc5fffe-c874-4979-a16b-307f7b3e2f53 kind: Event lastTimestamp: "2026-07-02T13:40:25Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:9f703da9f2f5afd108b6816a57feadccff9213d4d8437a63e732fa3fbaa3fa9d" already present on machine metadata: creationTimestamp: "2026-07-02T13:40:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:40:25Z" name: sequence-graph-2bc9c-6db49dd9b7-dfmqj.18be7cb66963c0f4 namespace: kserve-ci-e2e-test resourceVersion: "22188" uid: 15f3ff1f-19d9-45b9-9955-777e136f25ea reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:40:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-2bc9c} kind: Pod name: sequence-graph-2bc9c-6db49dd9b7-dfmqj namespace: kserve-ci-e2e-test resourceVersion: "22164" uid: 8dc5fffe-c874-4979-a16b-307f7b3e2f53 kind: Event lastTimestamp: "2026-07-02T13:40:25Z" message: 'Created container: sequence-graph-2bc9c' metadata: creationTimestamp: "2026-07-02T13:40:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:40:25Z" name: sequence-graph-2bc9c-6db49dd9b7-dfmqj.18be7cb66c7d15ed namespace: kserve-ci-e2e-test resourceVersion: "22191" uid: a62cdc68-2f2e-4133-baa2-25d7992d3591 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:40:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-2bc9c} kind: Pod name: sequence-graph-2bc9c-6db49dd9b7-dfmqj namespace: kserve-ci-e2e-test resourceVersion: "22164" uid: 8dc5fffe-c874-4979-a16b-307f7b3e2f53 kind: Event lastTimestamp: "2026-07-02T13:40:25Z" message: Started container sequence-graph-2bc9c metadata: creationTimestamp: "2026-07-02T13:40:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:40:25Z" name: sequence-graph-2bc9c-6db49dd9b7-dfmqj.18be7cb66d2e0fd4 namespace: kserve-ci-e2e-test resourceVersion: "22192" uid: a26843eb-9a13-4d71-b471-6e3ab2fddd6c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:40:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-2bc9c} kind: Pod name: sequence-graph-2bc9c-6db49dd9b7-dfmqj namespace: kserve-ci-e2e-test resourceVersion: "22164" uid: 8dc5fffe-c874-4979-a16b-307f7b3e2f53 kind: Event lastTimestamp: "2026-07-02T13:40:34Z" message: Stopping container sequence-graph-2bc9c metadata: creationTimestamp: "2026-07-02T13:40:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:40:35Z" name: sequence-graph-2bc9c-6db49dd9b7-dfmqj.18be7cb8a6c9022a namespace: kserve-ci-e2e-test resourceVersion: "22283" uid: 89feb81c-b95d-4e3b-8599-e96c6d899424 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:40:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-2bc9c} kind: Pod name: sequence-graph-2bc9c-6db49dd9b7-dfmqj namespace: kserve-ci-e2e-test resourceVersion: "22164" uid: 8dc5fffe-c874-4979-a16b-307f7b3e2f53 kind: Event lastTimestamp: "2026-07-02T13:41:02Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T13:40:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:41:02Z" name: sequence-graph-2bc9c-6db49dd9b7-dfmqj.18be7cb93894c867 namespace: kserve-ci-e2e-test resourceVersion: "22684" uid: 3be7450a-71f1-4091-9e35-18bcd8ff5dcb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:40:24Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-2bc9c-6db49dd9b7 namespace: kserve-ci-e2e-test resourceVersion: "22159" uid: 07ac1c48-3a54-40fc-9f37-8b9c9606c267 kind: Event lastTimestamp: "2026-07-02T13:40:24Z" message: 'Created pod: sequence-graph-2bc9c-6db49dd9b7-dfmqj' metadata: creationTimestamp: "2026-07-02T13:40:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:40:24Z" name: sequence-graph-2bc9c-6db49dd9b7.18be7cb64d35b10a namespace: kserve-ci-e2e-test resourceVersion: "22163" uid: d80aa34f-0ef9-481a-84ae-ed05bd04274b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:40:24Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-2bc9c namespace: kserve-ci-e2e-test resourceVersion: "22157" uid: 5d67ebc9-7190-464a-9e10-254b82fa8dc6 kind: Event lastTimestamp: "2026-07-02T13:40:24Z" message: Scaled up replica set sequence-graph-2bc9c-6db49dd9b7 from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:40:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:40:24Z" name: sequence-graph-2bc9c.18be7cb64c92fc4d namespace: kserve-ci-e2e-test resourceVersion: "22160" uid: 40ca16f2-9960-43d8-9a8a-2ebb3803d08c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:40:24Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-2bc9c namespace: kserve-ci-e2e-test resourceVersion: "22176" uid: c020cbff-9c7a-420f-b498-cd19329c733d kind: Event lastTimestamp: "2026-07-02T13:40:24Z" message: InferenceGraph [sequence-graph-2bc9c] is Ready metadata: creationTimestamp: "2026-07-02T13:40:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:40:24Z" name: sequence-graph-2bc9c.18be7cb65121f8de namespace: kserve-ci-e2e-test resourceVersion: "22178" uid: e448b0c6-1429-4977-9726-f28ddd4a16dc reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:41:35.186801Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-7b15c-657959bb75-qgx8p namespace: kserve-ci-e2e-test resourceVersion: "22971" uid: f3faf2de-97cd-44c2-9534-2c6d935c0447 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-7b15c-657959bb75-qgx8p to ip-10-0-133-122.ec2.internal metadata: creationTimestamp: "2026-07-02T13:41:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:41:35Z" name: sequence-graph-7b15c-657959bb75-qgx8p.18be7cc6aa3d8138 namespace: kserve-ci-e2e-test resourceVersion: "22976" uid: 5a3cc2f3-e8b5-4724-9eab-2c2fe648ccc2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7f4bb8bc8-mnq2w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:41:35Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-7b15c-657959bb75-qgx8p namespace: kserve-ci-e2e-test resourceVersion: "22980" uid: f3faf2de-97cd-44c2-9534-2c6d935c0447 kind: Event lastTimestamp: "2026-07-02T13:41:35Z" message: Add eth0 [10.134.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:41:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:41:35Z" name: sequence-graph-7b15c-657959bb75-qgx8p.18be7cc6c3ac261f namespace: kserve-ci-e2e-test resourceVersion: "22995" uid: 79b42d1e-50d9-42fe-90fe-4a471d98ebef reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:41:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-7b15c} kind: Pod name: sequence-graph-7b15c-657959bb75-qgx8p namespace: kserve-ci-e2e-test resourceVersion: "22972" uid: f3faf2de-97cd-44c2-9534-2c6d935c0447 kind: Event lastTimestamp: "2026-07-02T13:41:35Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:9f703da9f2f5afd108b6816a57feadccff9213d4d8437a63e732fa3fbaa3fa9d" already present on machine metadata: creationTimestamp: "2026-07-02T13:41:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:41:35Z" name: sequence-graph-7b15c-657959bb75-qgx8p.18be7cc6c4d54e37 namespace: kserve-ci-e2e-test resourceVersion: "22997" uid: 7cede23a-027b-421d-b990-d7353025213d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:41:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-7b15c} kind: Pod name: sequence-graph-7b15c-657959bb75-qgx8p namespace: kserve-ci-e2e-test resourceVersion: "22972" uid: f3faf2de-97cd-44c2-9534-2c6d935c0447 kind: Event lastTimestamp: "2026-07-02T13:41:35Z" message: 'Created container: sequence-graph-7b15c' metadata: creationTimestamp: "2026-07-02T13:41:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:41:35Z" name: sequence-graph-7b15c-657959bb75-qgx8p.18be7cc6c7d61a21 namespace: kserve-ci-e2e-test resourceVersion: "22998" uid: 1b4d1625-5024-4d81-9437-c8b86aaa8f88 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:41:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-7b15c} kind: Pod name: sequence-graph-7b15c-657959bb75-qgx8p namespace: kserve-ci-e2e-test resourceVersion: "22972" uid: f3faf2de-97cd-44c2-9534-2c6d935c0447 kind: Event lastTimestamp: "2026-07-02T13:41:35Z" message: Started container sequence-graph-7b15c metadata: creationTimestamp: "2026-07-02T13:41:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:41:35Z" name: sequence-graph-7b15c-657959bb75-qgx8p.18be7cc6c848d8e4 namespace: kserve-ci-e2e-test resourceVersion: "22999" uid: be501854-d0aa-41a1-9fc4-c7a13a47817d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:49:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-7b15c} kind: Pod name: sequence-graph-7b15c-657959bb75-qgx8p namespace: kserve-ci-e2e-test resourceVersion: "22972" uid: f3faf2de-97cd-44c2-9534-2c6d935c0447 kind: Event lastTimestamp: "2026-07-02T13:49:49Z" message: Stopping container sequence-graph-7b15c metadata: creationTimestamp: "2026-07-02T13:49:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:49:49Z" name: sequence-graph-7b15c-657959bb75-qgx8p.18be7d39d92f23b5 namespace: kserve-ci-e2e-test resourceVersion: "26862" uid: db768c3f-2d6c-4427-823a-cec4afb2f640 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:49:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-7b15c} kind: Pod name: sequence-graph-7b15c-657959bb75-qgx8p namespace: kserve-ci-e2e-test resourceVersion: "22972" uid: f3faf2de-97cd-44c2-9534-2c6d935c0447 kind: Event lastTimestamp: "2026-07-02T13:50:16Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T13:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:50:16Z" name: sequence-graph-7b15c-657959bb75-qgx8p.18be7d3a486694ef namespace: kserve-ci-e2e-test resourceVersion: "27262" uid: 94199a6a-e411-4443-9ace-9569d65c4eff reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:41:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-7b15c-657959bb75 namespace: kserve-ci-e2e-test resourceVersion: "22968" uid: 67afb5bb-026c-46c4-a8e8-2ecb0bf5bda8 kind: Event lastTimestamp: "2026-07-02T13:41:35Z" message: 'Created pod: sequence-graph-7b15c-657959bb75-qgx8p' metadata: creationTimestamp: "2026-07-02T13:41:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:41:35Z" name: sequence-graph-7b15c-657959bb75.18be7cc6a9e3ac70 namespace: kserve-ci-e2e-test resourceVersion: "22973" uid: ec8ab2a6-8bcc-4163-afd3-6b75a50e8201 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:41:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-7b15c namespace: kserve-ci-e2e-test resourceVersion: "22966" uid: a9e77ff4-13a5-46f3-a273-34453a6eacfc kind: Event lastTimestamp: "2026-07-02T13:41:35Z" message: Scaled up replica set sequence-graph-7b15c-657959bb75 from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:41:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:41:35Z" name: sequence-graph-7b15c.18be7cc6a929b303 namespace: kserve-ci-e2e-test resourceVersion: "22969" uid: 130e811a-0ec5-4170-90e1-e50eb15ebc59 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:41:35Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-7b15c namespace: kserve-ci-e2e-test resourceVersion: "22985" uid: 62e75e03-13a4-400b-bed9-0e4f592fd1cd kind: Event lastTimestamp: "2026-07-02T13:41:35Z" message: InferenceGraph [sequence-graph-7b15c] is Ready metadata: creationTimestamp: "2026-07-02T13:41:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:41:35Z" name: sequence-graph-7b15c.18be7cc6ade9da0e namespace: kserve-ci-e2e-test resourceVersion: "22987" uid: 5b7cd43c-3eec-4665-8ea6-702bc00cc5f7 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:41:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-7b15c namespace: kserve-ci-e2e-test resourceVersion: "22967" uid: c8b61aef-3190-4037-adb9-24035ffcf76d kind: Event lastTimestamp: "2026-07-02T13:42:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:41:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:42:20Z" name: sequence-graph-7b15c.18be7cca274872fb namespace: kserve-ci-e2e-test resourceVersion: "23320" uid: de1355ec-9c72-4879-a4b2-f80179e7d4f9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:41:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-7b15c namespace: kserve-ci-e2e-test resourceVersion: "22967" uid: c8b61aef-3190-4037-adb9-24035ffcf76d kind: Event lastTimestamp: "2026-07-02T13:42:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:41:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:42:20Z" name: sequence-graph-7b15c.18be7cca27a645b4 namespace: kserve-ci-e2e-test resourceVersion: "23321" uid: 1f7d4662-2ef8-4eb0-897a-9e61443ce60b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:31:10.089774Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-f369f-d9b69b6db-2jn9z namespace: kserve-ci-e2e-test resourceVersion: "17719" uid: 07fb52c1-e059-4a93-9aea-3468ebc711f6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-f369f-d9b69b6db-2jn9z to ip-10-0-133-122.ec2.internal metadata: creationTimestamp: "2026-07-02T13:31:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:31:10Z" name: sequence-graph-f369f-d9b69b6db-2jn9z.18be7c351f8d9ec8 namespace: kserve-ci-e2e-test resourceVersion: "17724" uid: 9f640235-ee95-4a87-a95c-60aaeda5dca4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7f4bb8bc8-mnq2w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:31:10Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-f369f-d9b69b6db-2jn9z namespace: kserve-ci-e2e-test resourceVersion: "17729" uid: 07fb52c1-e059-4a93-9aea-3468ebc711f6 kind: Event lastTimestamp: "2026-07-02T13:31:10Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:31:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:31:10Z" name: sequence-graph-f369f-d9b69b6db-2jn9z.18be7c3538b12377 namespace: kserve-ci-e2e-test resourceVersion: "17743" uid: bae2e040-eda2-41f6-b1b8-334267c16d03 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:31:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f369f} kind: Pod name: sequence-graph-f369f-d9b69b6db-2jn9z namespace: kserve-ci-e2e-test resourceVersion: "17721" uid: 07fb52c1-e059-4a93-9aea-3468ebc711f6 kind: Event lastTimestamp: "2026-07-02T13:31:10Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:9f703da9f2f5afd108b6816a57feadccff9213d4d8437a63e732fa3fbaa3fa9d" already present on machine metadata: creationTimestamp: "2026-07-02T13:31:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:31:10Z" name: sequence-graph-f369f-d9b69b6db-2jn9z.18be7c3539d8bdb1 namespace: kserve-ci-e2e-test resourceVersion: "17745" uid: fd12b738-85f4-4ea4-8a5b-03cfafa443b9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:31:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f369f} kind: Pod name: sequence-graph-f369f-d9b69b6db-2jn9z namespace: kserve-ci-e2e-test resourceVersion: "17721" uid: 07fb52c1-e059-4a93-9aea-3468ebc711f6 kind: Event lastTimestamp: "2026-07-02T13:31:10Z" message: 'Created container: sequence-graph-f369f' metadata: creationTimestamp: "2026-07-02T13:31:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:31:10Z" name: sequence-graph-f369f-d9b69b6db-2jn9z.18be7c353cceb7a1 namespace: kserve-ci-e2e-test resourceVersion: "17746" uid: 3c799ce8-6144-430f-b279-ae9d4655b067 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:31:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f369f} kind: Pod name: sequence-graph-f369f-d9b69b6db-2jn9z namespace: kserve-ci-e2e-test resourceVersion: "17721" uid: 07fb52c1-e059-4a93-9aea-3468ebc711f6 kind: Event lastTimestamp: "2026-07-02T13:31:10Z" message: Started container sequence-graph-f369f metadata: creationTimestamp: "2026-07-02T13:31:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:31:10Z" name: sequence-graph-f369f-d9b69b6db-2jn9z.18be7c353d3cb7db namespace: kserve-ci-e2e-test resourceVersion: "17747" uid: 3001fa30-e073-40b9-bf74-c493c8f4c380 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f369f} kind: Pod name: sequence-graph-f369f-d9b69b6db-2jn9z namespace: kserve-ci-e2e-test resourceVersion: "17721" uid: 07fb52c1-e059-4a93-9aea-3468ebc711f6 kind: Event lastTimestamp: "2026-07-02T13:39:24Z" message: Stopping container sequence-graph-f369f metadata: creationTimestamp: "2026-07-02T13:39:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:39:24Z" name: sequence-graph-f369f-d9b69b6db-2jn9z.18be7ca84cc8abc2 namespace: kserve-ci-e2e-test resourceVersion: "21322" uid: 14974769-64e0-440b-bd41-c5c7ba5b7a60 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:39:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f369f} kind: Pod name: sequence-graph-f369f-d9b69b6db-2jn9z namespace: kserve-ci-e2e-test resourceVersion: "17721" uid: 07fb52c1-e059-4a93-9aea-3468ebc711f6 kind: Event lastTimestamp: "2026-07-02T13:39:52Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T13:39:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:39:52Z" name: sequence-graph-f369f-d9b69b6db-2jn9z.18be7ca8d4d40697 namespace: kserve-ci-e2e-test resourceVersion: "21722" uid: d32a068b-a06a-474c-98dc-a897a9cd05ef reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:31:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-f369f-d9b69b6db namespace: kserve-ci-e2e-test resourceVersion: "17716" uid: 56fb276c-09a8-4dfb-a574-3598c5fa74c0 kind: Event lastTimestamp: "2026-07-02T13:31:10Z" message: 'Created pod: sequence-graph-f369f-d9b69b6db-2jn9z' metadata: creationTimestamp: "2026-07-02T13:31:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:31:10Z" name: sequence-graph-f369f-d9b69b6db.18be7c351f28a6ba namespace: kserve-ci-e2e-test resourceVersion: "17720" uid: a0a81a58-c126-432e-bc7e-64f93d702b50 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:31:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-f369f namespace: kserve-ci-e2e-test resourceVersion: "17714" uid: 194a1f5a-8403-48b1-bdfc-3104ef33d571 kind: Event lastTimestamp: "2026-07-02T13:31:10Z" message: Scaled up replica set sequence-graph-f369f-d9b69b6db from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:31:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:31:10Z" name: sequence-graph-f369f.18be7c351e6d90af namespace: kserve-ci-e2e-test resourceVersion: "17717" uid: b5a8016f-9766-459b-a6f2-86ea96b8306e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:31:10Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-f369f namespace: kserve-ci-e2e-test resourceVersion: "17732" uid: db5616de-5772-4b99-8fa6-7c73fe3922d8 kind: Event lastTimestamp: "2026-07-02T13:31:10Z" message: InferenceGraph [sequence-graph-f369f] is Ready metadata: creationTimestamp: "2026-07-02T13:31:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:31:10Z" name: sequence-graph-f369f.18be7c352323efda namespace: kserve-ci-e2e-test resourceVersion: "17734" uid: ae383bb2-72d1-4943-85a7-69f038a1fb25 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:31:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-f369f namespace: kserve-ci-e2e-test resourceVersion: "17715" uid: fcd4d691-89da-4591-b675-129d91b1ba03 kind: Event lastTimestamp: "2026-07-02T13:31:40Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:31:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:31:40Z" name: sequence-graph-f369f.18be7c389d227ad4 namespace: kserve-ci-e2e-test resourceVersion: "17962" uid: 2bb26a0d-6f89-4f4f-aeef-3710080e21e9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:31:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-f369f namespace: kserve-ci-e2e-test resourceVersion: "17715" uid: fcd4d691-89da-4591-b675-129d91b1ba03 kind: Event lastTimestamp: "2026-07-02T13:31:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:31:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:31:40Z" name: sequence-graph-f369f.18be7c389d7c8445 namespace: kserve-ci-e2e-test resourceVersion: "17963" uid: 3d720e3d-4e74-48ec-af08-49695fb1a7b0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:51:25.780984Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-03d0f-7b76878d5c-rzw2c namespace: kserve-ci-e2e-test resourceVersion: "28047" uid: d5462ee6-1101-4006-ac12-c3069ae275b7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-03d0f-7b76878d5c-rzw2c to ip-10-0-133-122.ec2.internal metadata: creationTimestamp: "2026-07-02T13:51:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:51:25Z" name: splitter-graph-03d0f-7b76878d5c-rzw2c.18be7d502c655acf namespace: kserve-ci-e2e-test resourceVersion: "28054" uid: c7571fd1-3516-4c1a-bd1a-183c5bd2bcee reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7f4bb8bc8-mnq2w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:51:26Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-03d0f-7b76878d5c-rzw2c namespace: kserve-ci-e2e-test resourceVersion: "28059" uid: d5462ee6-1101-4006-ac12-c3069ae275b7 kind: Event lastTimestamp: "2026-07-02T13:51:26Z" message: Add eth0 [10.134.0.49/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:51:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:51:26Z" name: splitter-graph-03d0f-7b76878d5c-rzw2c.18be7d50458edf2e namespace: kserve-ci-e2e-test resourceVersion: "28070" uid: e4b1f9cf-6c53-442e-88a1-b6bc69791e8c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:51:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-03d0f} kind: Pod name: splitter-graph-03d0f-7b76878d5c-rzw2c namespace: kserve-ci-e2e-test resourceVersion: "28049" uid: d5462ee6-1101-4006-ac12-c3069ae275b7 kind: Event lastTimestamp: "2026-07-02T13:51:26Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:9f703da9f2f5afd108b6816a57feadccff9213d4d8437a63e732fa3fbaa3fa9d" already present on machine metadata: creationTimestamp: "2026-07-02T13:51:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:51:26Z" name: splitter-graph-03d0f-7b76878d5c-rzw2c.18be7d5046bb199e namespace: kserve-ci-e2e-test resourceVersion: "28072" uid: d04c224e-b16f-4912-841c-98a5f8b18c69 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:51:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-03d0f} kind: Pod name: splitter-graph-03d0f-7b76878d5c-rzw2c namespace: kserve-ci-e2e-test resourceVersion: "28049" uid: d5462ee6-1101-4006-ac12-c3069ae275b7 kind: Event lastTimestamp: "2026-07-02T13:51:26Z" message: 'Created container: splitter-graph-03d0f' metadata: creationTimestamp: "2026-07-02T13:51:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:51:26Z" name: splitter-graph-03d0f-7b76878d5c-rzw2c.18be7d504996c46a namespace: kserve-ci-e2e-test resourceVersion: "28073" uid: 2559f439-dd79-4231-9822-e5ba9b7a7549 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:51:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-03d0f} kind: Pod name: splitter-graph-03d0f-7b76878d5c-rzw2c namespace: kserve-ci-e2e-test resourceVersion: "28049" uid: d5462ee6-1101-4006-ac12-c3069ae275b7 kind: Event lastTimestamp: "2026-07-02T13:51:26Z" message: Started container splitter-graph-03d0f metadata: creationTimestamp: "2026-07-02T13:51:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:51:26Z" name: splitter-graph-03d0f-7b76878d5c-rzw2c.18be7d504a057693 namespace: kserve-ci-e2e-test resourceVersion: "28074" uid: 76de7cd8-0119-432b-bce5-9e4b9b612a90 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:59:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-03d0f} kind: Pod name: splitter-graph-03d0f-7b76878d5c-rzw2c namespace: kserve-ci-e2e-test resourceVersion: "28049" uid: d5462ee6-1101-4006-ac12-c3069ae275b7 kind: Event lastTimestamp: "2026-07-02T13:59:40Z" message: Stopping container splitter-graph-03d0f metadata: creationTimestamp: "2026-07-02T13:59:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:59:40Z" name: splitter-graph-03d0f-7b76878d5c-rzw2c.18be7dc35451d627 namespace: kserve-ci-e2e-test resourceVersion: "31523" uid: 0fc55514-7920-4424-9a86-c231bfecf57e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:59:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-03d0f} kind: Pod name: splitter-graph-03d0f-7b76878d5c-rzw2c namespace: kserve-ci-e2e-test resourceVersion: "28049" uid: d5462ee6-1101-4006-ac12-c3069ae275b7 kind: Event lastTimestamp: "2026-07-02T14:00:07Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T13:59:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:00:07Z" name: splitter-graph-03d0f-7b76878d5c-rzw2c.18be7dc3ca72852e namespace: kserve-ci-e2e-test resourceVersion: "31752" uid: 6d4b94c0-d485-4f26-824c-0d99f177657c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:51:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-03d0f-7b76878d5c namespace: kserve-ci-e2e-test resourceVersion: "28044" uid: 1826d7d8-106c-4ae2-ba39-6075928e9865 kind: Event lastTimestamp: "2026-07-02T13:51:25Z" message: 'Created pod: splitter-graph-03d0f-7b76878d5c-rzw2c' metadata: creationTimestamp: "2026-07-02T13:51:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:51:25Z" name: splitter-graph-03d0f-7b76878d5c.18be7d502bb5b09f namespace: kserve-ci-e2e-test resourceVersion: "28050" uid: 1f3d4dd1-bb16-4aca-ad75-336f1c4769a5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:51:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-03d0f namespace: kserve-ci-e2e-test resourceVersion: "28042" uid: 1096dcf4-557f-4b1f-ab8e-94f0f09bf712 kind: Event lastTimestamp: "2026-07-02T13:51:25Z" message: Scaled up replica set splitter-graph-03d0f-7b76878d5c from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:51:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:51:25Z" name: splitter-graph-03d0f.18be7d502a236a7b namespace: kserve-ci-e2e-test resourceVersion: "28045" uid: 4e857f30-a456-45a8-9469-3814daec9576 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:51:25Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-03d0f namespace: kserve-ci-e2e-test resourceVersion: "28063" uid: 95aa5cac-4dc2-471f-aefe-5c2fb94cc6ae kind: Event lastTimestamp: "2026-07-02T13:51:25Z" message: InferenceGraph [splitter-graph-03d0f] is Ready metadata: creationTimestamp: "2026-07-02T13:51:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:51:25Z" name: splitter-graph-03d0f.18be7d50301ddae2 namespace: kserve-ci-e2e-test resourceVersion: "28065" uid: 60424105-d29c-41ba-90ab-8172a0791cf7 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:51:25Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-03d0f namespace: kserve-ci-e2e-test resourceVersion: "28034" uid: 95aa5cac-4dc2-471f-aefe-5c2fb94cc6ae kind: Event lastTimestamp: "2026-07-02T13:51:25Z" message: 'Failed to update status for InferenceGraph "splitter-graph-03d0f": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "splitter-graph-03d0f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:51:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:51:25Z" name: splitter-graph-03d0f.18be7d5031f5cea2 namespace: kserve-ci-e2e-test resourceVersion: "28066" uid: f9c2145e-7fe2-4e63-90b5-637603bec60b reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:51:25Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-03d0f namespace: kserve-ci-e2e-test resourceVersion: "28034" uid: 95aa5cac-4dc2-471f-aefe-5c2fb94cc6ae kind: Event lastTimestamp: "2026-07-02T13:51:25Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "splitter-graph-03d0f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:51:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:51:25Z" name: splitter-graph-03d0f.18be7d5031f617be namespace: kserve-ci-e2e-test resourceVersion: "28067" uid: a2e39164-d1c6-4864-b3a6-b3cc7efa1992 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:51:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-03d0f namespace: kserve-ci-e2e-test resourceVersion: "28043" uid: a9b33d62-39fa-4b36-83dc-26b7456ec88a kind: Event lastTimestamp: "2026-07-02T13:51:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:51:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:51:55Z" name: splitter-graph-03d0f.18be7d53a86374b6 namespace: kserve-ci-e2e-test resourceVersion: "28294" uid: 5fe0fc1c-2781-426f-8eae-170fdf79af62 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:51:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-03d0f namespace: kserve-ci-e2e-test resourceVersion: "28043" uid: a9b33d62-39fa-4b36-83dc-26b7456ec88a kind: Event lastTimestamp: "2026-07-02T13:51:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:51:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:51:55Z" name: splitter-graph-03d0f.18be7d53a8c039d1 namespace: kserve-ci-e2e-test resourceVersion: "28295" uid: ecd1ce34-93d9-401d-8cbf-0e9acf994b71 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:50:15.464539Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-9633b-986484644-tkj85 namespace: kserve-ci-e2e-test resourceVersion: "27226" uid: 3d2ad189-e2e8-400c-be38-cae7f926895d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-9633b-986484644-tkj85 to ip-10-0-133-122.ec2.internal metadata: creationTimestamp: "2026-07-02T13:50:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:50:15Z" name: splitter-graph-9633b-986484644-tkj85.18be7d3fcd3590bf namespace: kserve-ci-e2e-test resourceVersion: "27233" uid: 98ab5b53-5b69-4c91-9caf-d167afbe8583 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7f4bb8bc8-mnq2w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:50:15Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-9633b-986484644-tkj85 namespace: kserve-ci-e2e-test resourceVersion: "27229" uid: 3d2ad189-e2e8-400c-be38-cae7f926895d kind: Event lastTimestamp: "2026-07-02T13:50:15Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-9633b-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T13:50:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:50:15Z" name: splitter-graph-9633b-986484644-tkj85.18be7d3fd56b7be7 namespace: kserve-ci-e2e-test resourceVersion: "27248" uid: 9b946578-401a-4800-b32d-a1405891f74c reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:50:16Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-9633b-986484644-tkj85 namespace: kserve-ci-e2e-test resourceVersion: "27236" uid: 3d2ad189-e2e8-400c-be38-cae7f926895d kind: Event lastTimestamp: "2026-07-02T13:50:16Z" message: Add eth0 [10.134.0.46/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:50:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:50:16Z" name: splitter-graph-9633b-986484644-tkj85.18be7d400a77f05a namespace: kserve-ci-e2e-test resourceVersion: "27256" uid: 7ea5c174-f26f-40b7-b246-60c40336835d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:50:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-9633b} kind: Pod name: splitter-graph-9633b-986484644-tkj85 namespace: kserve-ci-e2e-test resourceVersion: "27229" uid: 3d2ad189-e2e8-400c-be38-cae7f926895d kind: Event lastTimestamp: "2026-07-02T13:50:16Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:9f703da9f2f5afd108b6816a57feadccff9213d4d8437a63e732fa3fbaa3fa9d" already present on machine metadata: creationTimestamp: "2026-07-02T13:50:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:50:16Z" name: splitter-graph-9633b-986484644-tkj85.18be7d400ba75c6f namespace: kserve-ci-e2e-test resourceVersion: "27258" uid: e7dcbf7e-560c-4fb1-b0f8-df9976f957fd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:50:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-9633b} kind: Pod name: splitter-graph-9633b-986484644-tkj85 namespace: kserve-ci-e2e-test resourceVersion: "27229" uid: 3d2ad189-e2e8-400c-be38-cae7f926895d kind: Event lastTimestamp: "2026-07-02T13:50:16Z" message: 'Created container: splitter-graph-9633b' metadata: creationTimestamp: "2026-07-02T13:50:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:50:16Z" name: splitter-graph-9633b-986484644-tkj85.18be7d400e823903 namespace: kserve-ci-e2e-test resourceVersion: "27259" uid: 6fbc23f7-1623-46c8-a73d-57b73c235447 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:50:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-9633b} kind: Pod name: splitter-graph-9633b-986484644-tkj85 namespace: kserve-ci-e2e-test resourceVersion: "27229" uid: 3d2ad189-e2e8-400c-be38-cae7f926895d kind: Event lastTimestamp: "2026-07-02T13:50:16Z" message: Started container splitter-graph-9633b metadata: creationTimestamp: "2026-07-02T13:50:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:50:16Z" name: splitter-graph-9633b-986484644-tkj85.18be7d400ef21f9b namespace: kserve-ci-e2e-test resourceVersion: "27260" uid: 89e84b64-6cee-4b71-b230-74ecf0aa6601 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:50:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-9633b} kind: Pod name: splitter-graph-9633b-986484644-tkj85 namespace: kserve-ci-e2e-test resourceVersion: "27229" uid: 3d2ad189-e2e8-400c-be38-cae7f926895d kind: Event lastTimestamp: "2026-07-02T13:50:25Z" message: Stopping container splitter-graph-9633b metadata: creationTimestamp: "2026-07-02T13:50:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:50:25Z" name: splitter-graph-9633b-986484644-tkj85.18be7d42265c5ef2 namespace: kserve-ci-e2e-test resourceVersion: "27356" uid: 0255305b-cc2d-4404-a52f-8541e88ca278 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:50:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-9633b} kind: Pod name: splitter-graph-9633b-986484644-tkj85 namespace: kserve-ci-e2e-test resourceVersion: "27229" uid: 3d2ad189-e2e8-400c-be38-cae7f926895d kind: Event lastTimestamp: "2026-07-02T13:50:53Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T13:50:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:50:53Z" name: splitter-graph-9633b-986484644-tkj85.18be7d42ba7c246d namespace: kserve-ci-e2e-test resourceVersion: "27756" uid: 6ce303a8-344b-4aa6-85f2-f3ba57b1db7b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:50:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-9633b-986484644 namespace: kserve-ci-e2e-test resourceVersion: "27223" uid: ee14b217-2d14-4caa-8423-4727679163bc kind: Event lastTimestamp: "2026-07-02T13:50:15Z" message: 'Created pod: splitter-graph-9633b-986484644-tkj85' metadata: creationTimestamp: "2026-07-02T13:50:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:50:15Z" name: splitter-graph-9633b-986484644.18be7d3fccb38f2c namespace: kserve-ci-e2e-test resourceVersion: "27228" uid: 99aaa61a-6b67-4233-91b8-2fc9b15961fb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:50:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-9633b namespace: kserve-ci-e2e-test resourceVersion: "27221" uid: 9b649679-735a-4be2-abfd-01857e303cd9 kind: Event lastTimestamp: "2026-07-02T13:50:15Z" message: Scaled up replica set splitter-graph-9633b-986484644 from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:50:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:50:15Z" name: splitter-graph-9633b.18be7d3fcc0173d9 namespace: kserve-ci-e2e-test resourceVersion: "27224" uid: 6cfab9aa-e289-4dfe-8051-f28884009804 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:50:15Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-9633b namespace: kserve-ci-e2e-test resourceVersion: "27243" uid: 5b41216d-8379-4545-9053-bf7492018eb4 kind: Event lastTimestamp: "2026-07-02T13:50:15Z" message: InferenceGraph [splitter-graph-9633b] is Ready metadata: creationTimestamp: "2026-07-02T13:50:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:50:15Z" name: splitter-graph-9633b.18be7d3fd03b50e2 namespace: kserve-ci-e2e-test resourceVersion: "27245" uid: ddad6a12-88ca-4c2e-a902-12a4dcddbce4 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:50:25.661248Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-03d0f-predictor-75bbcf79f5-hs9hm namespace: kserve-ci-e2e-test resourceVersion: "27386" uid: d83acafb-0f7a-479d-a051-9991dfabae58 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-03d0f-predictor-75bbcf79f5-hs9hm to ip-10-0-131-238.ec2.internal metadata: creationTimestamp: "2026-07-02T13:50:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:50:25Z" name: success-200-isvc-03d0f-predictor-75bbcf79f5-hs9hm.18be7d422cfb09ae namespace: kserve-ci-e2e-test resourceVersion: "27394" uid: 373eeea9-a208-402a-b418-b593db9e43a9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7f4bb8bc8-mnq2w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:50:26Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-03d0f-predictor-75bbcf79f5-hs9hm namespace: kserve-ci-e2e-test resourceVersion: "27400" uid: d83acafb-0f7a-479d-a051-9991dfabae58 kind: Event lastTimestamp: "2026-07-02T13:50:26Z" message: Add eth0 [10.132.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:50:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:50:26Z" name: success-200-isvc-03d0f-predictor-75bbcf79f5-hs9hm.18be7d424740d04e namespace: kserve-ci-e2e-test resourceVersion: "27455" uid: ac4dbcbc-4b6e-4a6d-83d0-88b942325162 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:50:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-03d0f-predictor-75bbcf79f5-hs9hm namespace: kserve-ci-e2e-test resourceVersion: "27390" uid: d83acafb-0f7a-479d-a051-9991dfabae58 kind: Event lastTimestamp: "2026-07-02T13:50:26Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1683" already present on machine metadata: creationTimestamp: "2026-07-02T13:50:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:50:26Z" name: success-200-isvc-03d0f-predictor-75bbcf79f5-hs9hm.18be7d424860fc24 namespace: kserve-ci-e2e-test resourceVersion: "27457" uid: 919cb625-7b50-46d1-8297-e4f4c705d5ed reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:50:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-03d0f-predictor-75bbcf79f5-hs9hm namespace: kserve-ci-e2e-test resourceVersion: "27390" uid: d83acafb-0f7a-479d-a051-9991dfabae58 kind: Event lastTimestamp: "2026-07-02T13:50:26Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T13:50:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:50:26Z" name: success-200-isvc-03d0f-predictor-75bbcf79f5-hs9hm.18be7d424b2ec7cd namespace: kserve-ci-e2e-test resourceVersion: "27458" uid: 75aa6cd3-add1-464f-8e72-808afd1cefc6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:50:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-03d0f-predictor-75bbcf79f5-hs9hm namespace: kserve-ci-e2e-test resourceVersion: "27390" uid: d83acafb-0f7a-479d-a051-9991dfabae58 kind: Event lastTimestamp: "2026-07-02T13:50:26Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T13:50:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:50:26Z" name: success-200-isvc-03d0f-predictor-75bbcf79f5-hs9hm.18be7d424bbaae92 namespace: kserve-ci-e2e-test resourceVersion: "27459" uid: b359d527-2acd-462c-9198-861707341242 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:50:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-03d0f-predictor-75bbcf79f5-hs9hm namespace: kserve-ci-e2e-test resourceVersion: "27390" uid: d83acafb-0f7a-479d-a051-9991dfabae58 kind: Event lastTimestamp: "2026-07-02T13:50:26Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T13:50:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:50:26Z" name: success-200-isvc-03d0f-predictor-75bbcf79f5-hs9hm.18be7d424bc4ed59 namespace: kserve-ci-e2e-test resourceVersion: "27460" uid: d9c43215-f20e-486f-88c6-9ef9b18d9407 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:50:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-03d0f-predictor-75bbcf79f5-hs9hm namespace: kserve-ci-e2e-test resourceVersion: "27390" uid: d83acafb-0f7a-479d-a051-9991dfabae58 kind: Event lastTimestamp: "2026-07-02T13:50:26Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T13:50:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:50:26Z" name: success-200-isvc-03d0f-predictor-75bbcf79f5-hs9hm.18be7d424eb508b6 namespace: kserve-ci-e2e-test resourceVersion: "27461" uid: 08d4c481-547c-46be-a5a7-7db8eec877e4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:50:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-03d0f-predictor-75bbcf79f5-hs9hm namespace: kserve-ci-e2e-test resourceVersion: "27390" uid: d83acafb-0f7a-479d-a051-9991dfabae58 kind: Event lastTimestamp: "2026-07-02T13:50:26Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:50:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:50:26Z" name: success-200-isvc-03d0f-predictor-75bbcf79f5-hs9hm.18be7d424f56afb8 namespace: kserve-ci-e2e-test resourceVersion: "27462" uid: d3f03e87-377e-4023-9bf0-c6807ea4ffca reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:50:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-03d0f-predictor-75bbcf79f5-hs9hm namespace: kserve-ci-e2e-test resourceVersion: "27390" uid: d83acafb-0f7a-479d-a051-9991dfabae58 kind: Event lastTimestamp: "2026-07-02T13:51:03Z" message: 'Readiness probe failed: dial tcp 10.132.0.43:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:50:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:51:03Z" name: success-200-isvc-03d0f-predictor-75bbcf79f5-hs9hm.18be7d42a94034d7 namespace: kserve-ci-e2e-test resourceVersion: "27846" uid: 75929f9f-1177-4126-b969-c5f0a2d29344 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:59:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-03d0f-predictor-75bbcf79f5-hs9hm namespace: kserve-ci-e2e-test resourceVersion: "27390" uid: d83acafb-0f7a-479d-a051-9991dfabae58 kind: Event lastTimestamp: "2026-07-02T13:59:40Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T13:59:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:59:40Z" name: success-200-isvc-03d0f-predictor-75bbcf79f5-hs9hm.18be7dc35b15076a namespace: kserve-ci-e2e-test resourceVersion: "31539" uid: 284efc7a-42e1-45c5-aa1b-e251983e27b3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:59:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-03d0f-predictor-75bbcf79f5-hs9hm namespace: kserve-ci-e2e-test resourceVersion: "27390" uid: d83acafb-0f7a-479d-a051-9991dfabae58 kind: Event lastTimestamp: "2026-07-02T13:59:40Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:59:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:59:40Z" name: success-200-isvc-03d0f-predictor-75bbcf79f5-hs9hm.18be7dc35b1571b8 namespace: kserve-ci-e2e-test resourceVersion: "31540" uid: f948a6f6-c08e-41ce-8740-4169393f4775 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:50:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-03d0f-predictor-75bbcf79f5 namespace: kserve-ci-e2e-test resourceVersion: "27369" uid: 75a39dae-147e-41cb-a8ed-24d2747a099f kind: Event lastTimestamp: "2026-07-02T13:50:25Z" message: 'Created pod: success-200-isvc-03d0f-predictor-75bbcf79f5-hs9hm' metadata: creationTimestamp: "2026-07-02T13:50:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:50:25Z" name: success-200-isvc-03d0f-predictor-75bbcf79f5.18be7d422c895add namespace: kserve-ci-e2e-test resourceVersion: "27389" uid: fa023af6-9a19-4242-8462-0a98f6dfd2dc reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:50:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-03d0f-predictor namespace: kserve-ci-e2e-test resourceVersion: "27366" uid: 9b7e0376-e038-454a-8b1b-cba15433a19d kind: Event lastTimestamp: "2026-07-02T13:50:25Z" message: Scaled up replica set success-200-isvc-03d0f-predictor-75bbcf79f5 from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:50:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:50:25Z" name: success-200-isvc-03d0f-predictor.18be7d422aec86fb namespace: kserve-ci-e2e-test resourceVersion: "27375" uid: da1e211f-9aa6-4f0e-9367-415638b44d0e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:50:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-03d0f-predictor namespace: kserve-ci-e2e-test resourceVersion: "27368" uid: cc61774c-6a3b-42b9-8a24-1185ea544593 kind: Event lastTimestamp: "2026-07-02T13:50:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:50:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:50:55Z" name: success-200-isvc-03d0f-predictor.18be7d45a93a72ca namespace: kserve-ci-e2e-test resourceVersion: "27778" uid: e987871d-bc6e-4a6c-bf78-ff06e80437fe reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:50:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-03d0f-predictor namespace: kserve-ci-e2e-test resourceVersion: "27368" uid: cc61774c-6a3b-42b9-8a24-1185ea544593 kind: Event lastTimestamp: "2026-07-02T13:50:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:50:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:50:55Z" name: success-200-isvc-03d0f-predictor.18be7d45a99388b1 namespace: kserve-ci-e2e-test resourceVersion: "27779" uid: f6444856-3e4d-42b5-9dcc-2c5d84380153 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:51:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-03d0f-predictor namespace: kserve-ci-e2e-test resourceVersion: "27626" uid: cc61774c-6a3b-42b9-8a24-1185ea544593 kind: Event lastTimestamp: "2026-07-02T13:51:40Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:51:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:51:40Z" name: success-200-isvc-03d0f-predictor.18be7d4ca701873d namespace: kserve-ci-e2e-test resourceVersion: "28189" uid: de06f47d-6c8f-415d-a9cb-21021ff03537 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:51:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-03d0f-predictor namespace: kserve-ci-e2e-test resourceVersion: "27626" uid: cc61774c-6a3b-42b9-8a24-1185ea544593 kind: Event lastTimestamp: "2026-07-02T13:51:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:51:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:51:40Z" name: success-200-isvc-03d0f-predictor.18be7d4ca75f889a namespace: kserve-ci-e2e-test resourceVersion: "28190" uid: d87b9ab3-2a21-45cd-adf4-c236dd434b40 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T13:50:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-03d0f namespace: kserve-ci-e2e-test resourceVersion: "27360" uid: 6b1466fb-094d-4b3e-b16f-f084cd3ea49b kind: Event lastTimestamp: "2026-07-02T13:51:13Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T13:50:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:51:13Z" name: success-200-isvc-03d0f.18be7d422aeeacf8 namespace: kserve-ci-e2e-test resourceVersion: "27952" uid: 52023ac9-f01b-41cd-9dbd-861a3a936e3e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:51:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-03d0f namespace: kserve-ci-e2e-test resourceVersion: "27950" uid: 6b1466fb-094d-4b3e-b16f-f084cd3ea49b kind: Event lastTimestamp: "2026-07-02T13:51:13Z" message: InferenceService [success-200-isvc-03d0f] is Ready metadata: creationTimestamp: "2026-07-02T13:51:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:51:13Z" name: success-200-isvc-03d0f.18be7d4d63751fa6 namespace: kserve-ci-e2e-test resourceVersion: "27951" uid: 46c293da-985d-4209-bbf3-38bb83fc5cf1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:39:24.903913Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-2bc9c-predictor-59ff898c87-zfgzl namespace: kserve-ci-e2e-test resourceVersion: "21368" uid: b32c919d-6838-47bd-ad43-56804a8f3937 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-2bc9c-predictor-59ff898c87-zfgzl to ip-10-0-131-238.ec2.internal metadata: creationTimestamp: "2026-07-02T13:39:24Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:39:24Z" name: success-200-isvc-2bc9c-predictor-59ff898c87-zfgzl.18be7ca854c6384e namespace: kserve-ci-e2e-test resourceVersion: "21375" uid: a8bcc187-622a-4dfa-b249-b5f990cc3c42 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7f4bb8bc8-mnq2w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:25Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-2bc9c-predictor-59ff898c87-zfgzl namespace: kserve-ci-e2e-test resourceVersion: "21370" uid: b32c919d-6838-47bd-ad43-56804a8f3937 kind: Event lastTimestamp: "2026-07-02T13:39:25Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-2bc9c-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T13:39:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:39:25Z" name: success-200-isvc-2bc9c-predictor-59ff898c87-zfgzl.18be7ca85e91fca1 namespace: kserve-ci-e2e-test resourceVersion: "21413" uid: 73ab93dc-b8bf-42e5-92b0-1d75d35313a2 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:25Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-2bc9c-predictor-59ff898c87-zfgzl namespace: kserve-ci-e2e-test resourceVersion: "21383" uid: b32c919d-6838-47bd-ad43-56804a8f3937 kind: Event lastTimestamp: "2026-07-02T13:39:25Z" message: Add eth0 [10.132.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:39:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:39:25Z" name: success-200-isvc-2bc9c-predictor-59ff898c87-zfgzl.18be7ca892baf40a namespace: kserve-ci-e2e-test resourceVersion: "21430" uid: e6efa92c-b8f3-418f-a35a-ff57dd8b3b4a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2bc9c-predictor-59ff898c87-zfgzl namespace: kserve-ci-e2e-test resourceVersion: "21370" uid: b32c919d-6838-47bd-ad43-56804a8f3937 kind: Event lastTimestamp: "2026-07-02T13:39:25Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1683" already present on machine metadata: creationTimestamp: "2026-07-02T13:39:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:39:25Z" name: success-200-isvc-2bc9c-predictor-59ff898c87-zfgzl.18be7ca894006765 namespace: kserve-ci-e2e-test resourceVersion: "21432" uid: 1084f1e3-0093-4c91-a97e-ea9355de7cd3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2bc9c-predictor-59ff898c87-zfgzl namespace: kserve-ci-e2e-test resourceVersion: "21370" uid: b32c919d-6838-47bd-ad43-56804a8f3937 kind: Event lastTimestamp: "2026-07-02T13:39:26Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T13:39:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:39:26Z" name: success-200-isvc-2bc9c-predictor-59ff898c87-zfgzl.18be7ca89768f636 namespace: kserve-ci-e2e-test resourceVersion: "21434" uid: 55ed40ba-d736-470a-911e-e2869d21ff11 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2bc9c-predictor-59ff898c87-zfgzl namespace: kserve-ci-e2e-test resourceVersion: "21370" uid: b32c919d-6838-47bd-ad43-56804a8f3937 kind: Event lastTimestamp: "2026-07-02T13:39:26Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T13:39:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:39:26Z" name: success-200-isvc-2bc9c-predictor-59ff898c87-zfgzl.18be7ca897da0fe1 namespace: kserve-ci-e2e-test resourceVersion: "21435" uid: 57a7db22-4e00-4d0e-853f-f346b8960354 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2bc9c-predictor-59ff898c87-zfgzl namespace: kserve-ci-e2e-test resourceVersion: "21370" uid: b32c919d-6838-47bd-ad43-56804a8f3937 kind: Event lastTimestamp: "2026-07-02T13:39:26Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T13:39:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:39:26Z" name: success-200-isvc-2bc9c-predictor-59ff898c87-zfgzl.18be7ca897e42593 namespace: kserve-ci-e2e-test resourceVersion: "21437" uid: 193c2471-38c1-473b-866a-46b5f3f385ba reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2bc9c-predictor-59ff898c87-zfgzl namespace: kserve-ci-e2e-test resourceVersion: "21370" uid: b32c919d-6838-47bd-ad43-56804a8f3937 kind: Event lastTimestamp: "2026-07-02T13:39:26Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T13:39:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:39:26Z" name: success-200-isvc-2bc9c-predictor-59ff898c87-zfgzl.18be7ca89ae1caf0 namespace: kserve-ci-e2e-test resourceVersion: "21439" uid: 35e7221e-c228-47dd-9100-fd330db36288 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2bc9c-predictor-59ff898c87-zfgzl namespace: kserve-ci-e2e-test resourceVersion: "21370" uid: b32c919d-6838-47bd-ad43-56804a8f3937 kind: Event lastTimestamp: "2026-07-02T13:39:26Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:39:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:39:26Z" name: success-200-isvc-2bc9c-predictor-59ff898c87-zfgzl.18be7ca89b503afe namespace: kserve-ci-e2e-test resourceVersion: "21440" uid: ce81f6a7-cc76-4bfd-abbd-3869a22b3466 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:39:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2bc9c-predictor-59ff898c87-zfgzl namespace: kserve-ci-e2e-test resourceVersion: "21370" uid: b32c919d-6838-47bd-ad43-56804a8f3937 kind: Event lastTimestamp: "2026-07-02T13:40:03Z" message: 'Readiness probe failed: dial tcp 10.132.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:39:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:40:03Z" name: success-200-isvc-2bc9c-predictor-59ff898c87-zfgzl.18be7ca8f44a846e namespace: kserve-ci-e2e-test resourceVersion: "21962" uid: 3e01257d-a5b9-462d-8765-5f46ad64fbc9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:40:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2bc9c-predictor-59ff898c87-zfgzl namespace: kserve-ci-e2e-test resourceVersion: "21370" uid: b32c919d-6838-47bd-ad43-56804a8f3937 kind: Event lastTimestamp: "2026-07-02T13:40:35Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T13:40:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:40:35Z" name: success-200-isvc-2bc9c-predictor-59ff898c87-zfgzl.18be7cb8ae8579fb namespace: kserve-ci-e2e-test resourceVersion: "22340" uid: f2fa64c4-197f-4c9c-99f6-a20edf3c607f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:40:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2bc9c-predictor-59ff898c87-zfgzl namespace: kserve-ci-e2e-test resourceVersion: "21370" uid: b32c919d-6838-47bd-ad43-56804a8f3937 kind: Event lastTimestamp: "2026-07-02T13:40:35Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:40:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:40:35Z" name: success-200-isvc-2bc9c-predictor-59ff898c87-zfgzl.18be7cb8ae85bb77 namespace: kserve-ci-e2e-test resourceVersion: "22342" uid: edca3e6f-d82c-4e4c-93fa-3dfa55961279 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:24Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-2bc9c-predictor-59ff898c87 namespace: kserve-ci-e2e-test resourceVersion: "21356" uid: 3512153f-7e07-4d5b-ae65-c4262250469b kind: Event lastTimestamp: "2026-07-02T13:39:24Z" message: 'Created pod: success-200-isvc-2bc9c-predictor-59ff898c87-zfgzl' metadata: creationTimestamp: "2026-07-02T13:39:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:39:24Z" name: success-200-isvc-2bc9c-predictor-59ff898c87.18be7ca85469b965 namespace: kserve-ci-e2e-test resourceVersion: "21369" uid: 1306f5ae-67dd-4ad1-b31d-db2e7a487c1d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:24Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-2bc9c-predictor namespace: kserve-ci-e2e-test resourceVersion: "21349" uid: 640f6480-ad01-43e6-8fec-b032068a6f43 kind: Event lastTimestamp: "2026-07-02T13:39:24Z" message: Scaled up replica set success-200-isvc-2bc9c-predictor-59ff898c87 from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:39:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:39:24Z" name: success-200-isvc-2bc9c-predictor.18be7ca852b7a84a namespace: kserve-ci-e2e-test resourceVersion: "21358" uid: ee36a1f8-44d2-4e96-9506-72f7d66677e5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:39:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2bc9c-predictor namespace: kserve-ci-e2e-test resourceVersion: "21350" uid: 350b729e-2950-4079-8c98-6a67f16c8508 kind: Event lastTimestamp: "2026-07-02T13:39:54Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:39:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:39:54Z" name: success-200-isvc-2bc9c-predictor.18be7cabd0fee629 namespace: kserve-ci-e2e-test resourceVersion: "21750" uid: e2cc89ce-9fb0-4bdf-9d1b-8925e7cbf08f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:39:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2bc9c-predictor namespace: kserve-ci-e2e-test resourceVersion: "21350" uid: 350b729e-2950-4079-8c98-6a67f16c8508 kind: Event lastTimestamp: "2026-07-02T13:39:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:39:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:39:54Z" name: success-200-isvc-2bc9c-predictor.18be7cabd15bfe3c namespace: kserve-ci-e2e-test resourceVersion: "21751" uid: 971aca26-7832-4d06-bd5c-3012cffff7d0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:40:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2bc9c-predictor namespace: kserve-ci-e2e-test resourceVersion: "21587" uid: 350b729e-2950-4079-8c98-6a67f16c8508 kind: Event lastTimestamp: "2026-07-02T13:40:24Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:40:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:40:24Z" name: success-200-isvc-2bc9c-predictor.18be7cb2ce8d8307 namespace: kserve-ci-e2e-test resourceVersion: "22172" uid: 6e77239b-4b22-4f8d-ac18-f8c7184e3a2b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:40:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2bc9c-predictor namespace: kserve-ci-e2e-test resourceVersion: "21587" uid: 350b729e-2950-4079-8c98-6a67f16c8508 kind: Event lastTimestamp: "2026-07-02T13:40:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:40:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:40:24Z" name: success-200-isvc-2bc9c-predictor.18be7cb2cee9b1bd namespace: kserve-ci-e2e-test resourceVersion: "22174" uid: 95f764a6-1e3b-4f35-9406-1adff4ee7903 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:39:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2bc9c namespace: kserve-ci-e2e-test resourceVersion: "21327" uid: 412ddc21-0aca-42cb-ab28-f91531e2e8e4 kind: Event lastTimestamp: "2026-07-02T13:40:13Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T13:39:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:40:13Z" name: success-200-isvc-2bc9c.18be7ca852abf110 namespace: kserve-ci-e2e-test resourceVersion: "22057" uid: 96d6bab7-6f11-463d-8a80-f0a4805c9ea2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:40:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2bc9c namespace: kserve-ci-e2e-test resourceVersion: "22055" uid: 412ddc21-0aca-42cb-ab28-f91531e2e8e4 kind: Event lastTimestamp: "2026-07-02T13:40:13Z" message: InferenceService [success-200-isvc-2bc9c] is Ready metadata: creationTimestamp: "2026-07-02T13:40:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:40:13Z" name: success-200-isvc-2bc9c.18be7cb3ae85d9c7 namespace: kserve-ci-e2e-test resourceVersion: "22056" uid: 1b15e9c1-e5ec-4b87-8667-eda86a8a9414 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:29:30.271682Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-383ed-predictor-55fbb9d4ff-mc9t2 namespace: kserve-ci-e2e-test resourceVersion: "16552" uid: e7a6b0e7-1421-4980-b853-47d4df282a10 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-383ed-predictor-55fbb9d4ff-mc9t2 to ip-10-0-131-238.ec2.internal metadata: creationTimestamp: "2026-07-02T13:29:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:29:30Z" name: success-200-isvc-383ed-predictor-55fbb9d4ff-mc9t2.18be7c1de1ee6d2b namespace: kserve-ci-e2e-test resourceVersion: "16557" uid: 21a3fc3f-7d34-4f17-b822-9b97cb882299 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7f4bb8bc8-mnq2w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:29:30Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-383ed-predictor-55fbb9d4ff-mc9t2 namespace: kserve-ci-e2e-test resourceVersion: "16569" uid: e7a6b0e7-1421-4980-b853-47d4df282a10 kind: Event lastTimestamp: "2026-07-02T13:29:30Z" message: Add eth0 [10.132.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:29:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:29:30Z" name: success-200-isvc-383ed-predictor-55fbb9d4ff-mc9t2.18be7c1dfbb2599e namespace: kserve-ci-e2e-test resourceVersion: "16614" uid: 2a4705b7-d44e-4859-8d7c-112d99c24a9a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:29:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-383ed-predictor-55fbb9d4ff-mc9t2 namespace: kserve-ci-e2e-test resourceVersion: "16554" uid: e7a6b0e7-1421-4980-b853-47d4df282a10 kind: Event lastTimestamp: "2026-07-02T13:29:30Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1683" already present on machine metadata: creationTimestamp: "2026-07-02T13:29:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:29:30Z" name: success-200-isvc-383ed-predictor-55fbb9d4ff-mc9t2.18be7c1dfcda64c0 namespace: kserve-ci-e2e-test resourceVersion: "16616" uid: 41e7d24d-a0ac-4f62-8437-365750cf4aa1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:29:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-383ed-predictor-55fbb9d4ff-mc9t2 namespace: kserve-ci-e2e-test resourceVersion: "16554" uid: e7a6b0e7-1421-4980-b853-47d4df282a10 kind: Event lastTimestamp: "2026-07-02T13:29:30Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T13:29:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:29:30Z" name: success-200-isvc-383ed-predictor-55fbb9d4ff-mc9t2.18be7c1e00102915 namespace: kserve-ci-e2e-test resourceVersion: "16622" uid: ec3b78db-9da5-4475-98dc-052a76c4c7bb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:29:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-383ed-predictor-55fbb9d4ff-mc9t2 namespace: kserve-ci-e2e-test resourceVersion: "16554" uid: e7a6b0e7-1421-4980-b853-47d4df282a10 kind: Event lastTimestamp: "2026-07-02T13:29:30Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T13:29:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:29:30Z" name: success-200-isvc-383ed-predictor-55fbb9d4ff-mc9t2.18be7c1e00866000 namespace: kserve-ci-e2e-test resourceVersion: "16623" uid: 0191ba3b-b055-4fb1-a5b1-29cbfe7a05ee reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:29:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-383ed-predictor-55fbb9d4ff-mc9t2 namespace: kserve-ci-e2e-test resourceVersion: "16554" uid: e7a6b0e7-1421-4980-b853-47d4df282a10 kind: Event lastTimestamp: "2026-07-02T13:29:30Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T13:29:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:29:30Z" name: success-200-isvc-383ed-predictor-55fbb9d4ff-mc9t2.18be7c1e0090435e namespace: kserve-ci-e2e-test resourceVersion: "16624" uid: 18a4a7c0-4dfc-4a50-9f60-442b92deb44b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:29:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-383ed-predictor-55fbb9d4ff-mc9t2 namespace: kserve-ci-e2e-test resourceVersion: "16554" uid: e7a6b0e7-1421-4980-b853-47d4df282a10 kind: Event lastTimestamp: "2026-07-02T13:29:30Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T13:29:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:29:30Z" name: success-200-isvc-383ed-predictor-55fbb9d4ff-mc9t2.18be7c1e0352be46 namespace: kserve-ci-e2e-test resourceVersion: "16628" uid: 53e40680-2bd3-4fe8-9029-965737f0dbae reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:29:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-383ed-predictor-55fbb9d4ff-mc9t2 namespace: kserve-ci-e2e-test resourceVersion: "16554" uid: e7a6b0e7-1421-4980-b853-47d4df282a10 kind: Event lastTimestamp: "2026-07-02T13:29:30Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:29:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:29:30Z" name: success-200-isvc-383ed-predictor-55fbb9d4ff-mc9t2.18be7c1e03cb5bac namespace: kserve-ci-e2e-test resourceVersion: "16629" uid: d69de109-d8c9-4b26-a0e6-d35b31748b45 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-02T13:29:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-383ed-predictor-55fbb9d4ff-mc9t2 namespace: kserve-ci-e2e-test resourceVersion: "16554" uid: e7a6b0e7-1421-4980-b853-47d4df282a10 kind: Event lastTimestamp: "2026-07-02T13:38:47Z" message: 'Readiness probe failed: dial tcp 10.132.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:29:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:38:47Z" name: success-200-isvc-383ed-predictor-55fbb9d4ff-mc9t2.18be7c1e381dcc57 namespace: kserve-ci-e2e-test resourceVersion: "21010" uid: dc99bbcb-2f82-4988-8320-79035911d6a6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:38:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-383ed-predictor-55fbb9d4ff-mc9t2 namespace: kserve-ci-e2e-test resourceVersion: "16554" uid: e7a6b0e7-1421-4980-b853-47d4df282a10 kind: Event lastTimestamp: "2026-07-02T13:38:45Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T13:38:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:38:45Z" name: success-200-isvc-383ed-predictor-55fbb9d4ff-mc9t2.18be7c9f1233bb80 namespace: kserve-ci-e2e-test resourceVersion: "20897" uid: 35d2a02f-80a6-4142-a7b7-4147be1c684c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:38:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-383ed-predictor-55fbb9d4ff-mc9t2 namespace: kserve-ci-e2e-test resourceVersion: "16554" uid: e7a6b0e7-1421-4980-b853-47d4df282a10 kind: Event lastTimestamp: "2026-07-02T13:38:45Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:38:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:38:45Z" name: success-200-isvc-383ed-predictor-55fbb9d4ff-mc9t2.18be7c9f123444a1 namespace: kserve-ci-e2e-test resourceVersion: "20901" uid: eb6818e6-6586-4341-9742-45bdf98b3e9c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:38:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-383ed-predictor-55fbb9d4ff-mc9t2 namespace: kserve-ci-e2e-test resourceVersion: "16554" uid: e7a6b0e7-1421-4980-b853-47d4df282a10 kind: Event lastTimestamp: "2026-07-02T13:38:47Z" message: 'Readiness probe failed: Get "https://10.132.0.35:8643/healthz": dial tcp 10.132.0.35:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:38:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:38:47Z" name: success-200-isvc-383ed-predictor-55fbb9d4ff-mc9t2.18be7c9fac8a2466 namespace: kserve-ci-e2e-test resourceVersion: "21009" uid: 4b71e96b-27af-4240-8e82-e8dc9173c1d4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:29:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-383ed-predictor-55fbb9d4ff namespace: kserve-ci-e2e-test resourceVersion: "16529" uid: 0f04ce3f-77bb-4871-8552-a8d8c39f4a4b kind: Event lastTimestamp: "2026-07-02T13:29:30Z" message: 'Created pod: success-200-isvc-383ed-predictor-55fbb9d4ff-mc9t2' metadata: creationTimestamp: "2026-07-02T13:29:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:29:30Z" name: success-200-isvc-383ed-predictor-55fbb9d4ff.18be7c1de169ea18 namespace: kserve-ci-e2e-test resourceVersion: "16553" uid: 288169d1-8d67-45e2-aa24-4dfdd4d278f5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:29:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-383ed-predictor namespace: kserve-ci-e2e-test resourceVersion: "16526" uid: 7c223a1a-1b16-4dc1-8177-e3918737ebfa kind: Event lastTimestamp: "2026-07-02T13:29:30Z" message: Scaled up replica set success-200-isvc-383ed-predictor-55fbb9d4ff from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:29:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:29:30Z" name: success-200-isvc-383ed-predictor.18be7c1dde629a78 namespace: kserve-ci-e2e-test resourceVersion: "16535" uid: d73f8a5b-1c40-40f4-9032-cbcdedbd8be8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:29:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-383ed-predictor namespace: kserve-ci-e2e-test resourceVersion: "16528" uid: 63a35a24-08c3-49bf-93d3-306d5d4944e3 kind: Event lastTimestamp: "2026-07-02T13:30:00Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:29:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:30:00Z" name: success-200-isvc-383ed-predictor.18be7c215cc3dee0 namespace: kserve-ci-e2e-test resourceVersion: "16908" uid: 68a641b2-c6ea-46c9-9f8d-717f659fe21f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:29:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-383ed-predictor namespace: kserve-ci-e2e-test resourceVersion: "16528" uid: 63a35a24-08c3-49bf-93d3-306d5d4944e3 kind: Event lastTimestamp: "2026-07-02T13:30:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:29:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:30:00Z" name: success-200-isvc-383ed-predictor.18be7c215d243728 namespace: kserve-ci-e2e-test resourceVersion: "16909" uid: 1320848e-81a6-4e95-a84e-78cbb0731ea8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T13:30:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-383ed-predictor namespace: kserve-ci-e2e-test resourceVersion: "16761" uid: 63a35a24-08c3-49bf-93d3-306d5d4944e3 kind: Event lastTimestamp: "2026-07-02T13:31:00Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:30:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:31:00Z" name: success-200-isvc-383ed-predictor.18be7c285a887c7b namespace: kserve-ci-e2e-test resourceVersion: "17633" uid: 18170a93-2e50-47be-9e1b-8ae4d3e04ecc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T13:30:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-383ed-predictor namespace: kserve-ci-e2e-test resourceVersion: "16761" uid: 63a35a24-08c3-49bf-93d3-306d5d4944e3 kind: Event lastTimestamp: "2026-07-02T13:31:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:30:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:31:00Z" name: success-200-isvc-383ed-predictor.18be7c285aca95a0 namespace: kserve-ci-e2e-test resourceVersion: "17634" uid: 1dc5ebd6-d806-4eea-955e-8b17477f0e43 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:29:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-383ed namespace: kserve-ci-e2e-test resourceVersion: "16520" uid: 7425b641-916a-4483-8705-7932b284c540 kind: Event lastTimestamp: "2026-07-02T13:30:17Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T13:29:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:30:17Z" name: success-200-isvc-383ed.18be7c1dde738bfb namespace: kserve-ci-e2e-test resourceVersion: "17238" uid: 390b2b33-25d4-4230-9886-7748ae799f0a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:30:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-383ed namespace: kserve-ci-e2e-test resourceVersion: "17236" uid: 7425b641-916a-4483-8705-7932b284c540 kind: Event lastTimestamp: "2026-07-02T13:30:17Z" message: InferenceService [success-200-isvc-383ed] is Ready metadata: creationTimestamp: "2026-07-02T13:30:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:30:17Z" name: success-200-isvc-383ed.18be7c28f25b9789 namespace: kserve-ci-e2e-test resourceVersion: "17237" uid: ab2710e5-7fbf-41ed-8fc4-1045f68c9a6c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:38:45.182386Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6c5e8-predictor-866c9df647-kc54f namespace: kserve-ci-e2e-test resourceVersion: "20904" uid: 328ec472-00f3-45f9-ab8b-9ae649b29ee8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-6c5e8-predictor-866c9df647-kc54f to ip-10-0-131-238.ec2.internal metadata: creationTimestamp: "2026-07-02T13:38:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:38:45Z" name: success-200-isvc-6c5e8-predictor-866c9df647-kc54f.18be7c9f152fde43 namespace: kserve-ci-e2e-test resourceVersion: "20912" uid: 1f6e8037-8eae-478c-9ff4-ebc622fe7013 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7f4bb8bc8-mnq2w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:38:45Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6c5e8-predictor-866c9df647-kc54f namespace: kserve-ci-e2e-test resourceVersion: "20908" uid: 328ec472-00f3-45f9-ab8b-9ae649b29ee8 kind: Event lastTimestamp: "2026-07-02T13:38:45Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-6c5e8-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T13:38:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:38:45Z" name: success-200-isvc-6c5e8-predictor-866c9df647-kc54f.18be7c9f242941be namespace: kserve-ci-e2e-test resourceVersion: "20963" uid: ee5e4043-8851-46e2-ac98-0a34fab7170f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:38:46Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6c5e8-predictor-866c9df647-kc54f namespace: kserve-ci-e2e-test resourceVersion: "20922" uid: 328ec472-00f3-45f9-ab8b-9ae649b29ee8 kind: Event lastTimestamp: "2026-07-02T13:38:46Z" message: Add eth0 [10.132.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:38:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:38:46Z" name: success-200-isvc-6c5e8-predictor-866c9df647-kc54f.18be7c9f52b59a1c namespace: kserve-ci-e2e-test resourceVersion: "20983" uid: e4af4534-c67b-43af-91ff-49c081b9d965 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:38:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6c5e8-predictor-866c9df647-kc54f namespace: kserve-ci-e2e-test resourceVersion: "20908" uid: 328ec472-00f3-45f9-ab8b-9ae649b29ee8 kind: Event lastTimestamp: "2026-07-02T13:38:46Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1683" already present on machine metadata: creationTimestamp: "2026-07-02T13:38:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:38:46Z" name: success-200-isvc-6c5e8-predictor-866c9df647-kc54f.18be7c9f53f24022 namespace: kserve-ci-e2e-test resourceVersion: "20985" uid: 39dedfa5-b33b-492f-b35f-f86f049a537a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:38:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6c5e8-predictor-866c9df647-kc54f namespace: kserve-ci-e2e-test resourceVersion: "20908" uid: 328ec472-00f3-45f9-ab8b-9ae649b29ee8 kind: Event lastTimestamp: "2026-07-02T13:38:46Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T13:38:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:38:46Z" name: success-200-isvc-6c5e8-predictor-866c9df647-kc54f.18be7c9f57376f15 namespace: kserve-ci-e2e-test resourceVersion: "20986" uid: e1c99054-7796-475b-99e3-181a3b234eaf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:38:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6c5e8-predictor-866c9df647-kc54f namespace: kserve-ci-e2e-test resourceVersion: "20908" uid: 328ec472-00f3-45f9-ab8b-9ae649b29ee8 kind: Event lastTimestamp: "2026-07-02T13:38:46Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T13:38:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:38:46Z" name: success-200-isvc-6c5e8-predictor-866c9df647-kc54f.18be7c9f57a88a51 namespace: kserve-ci-e2e-test resourceVersion: "20987" uid: 583b2dfb-47fd-4dd3-a951-edd124800a9b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:38:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6c5e8-predictor-866c9df647-kc54f namespace: kserve-ci-e2e-test resourceVersion: "20908" uid: 328ec472-00f3-45f9-ab8b-9ae649b29ee8 kind: Event lastTimestamp: "2026-07-02T13:38:46Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T13:38:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:38:46Z" name: success-200-isvc-6c5e8-predictor-866c9df647-kc54f.18be7c9f57b247bc namespace: kserve-ci-e2e-test resourceVersion: "20988" uid: 26382658-cf74-40ef-97aa-34b60d660fe2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:38:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6c5e8-predictor-866c9df647-kc54f namespace: kserve-ci-e2e-test resourceVersion: "20908" uid: 328ec472-00f3-45f9-ab8b-9ae649b29ee8 kind: Event lastTimestamp: "2026-07-02T13:38:46Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T13:38:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:38:46Z" name: success-200-isvc-6c5e8-predictor-866c9df647-kc54f.18be7c9f5b604eee namespace: kserve-ci-e2e-test resourceVersion: "20991" uid: 25e1de2b-6cf6-452f-933f-1a4d98d384e2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:38:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6c5e8-predictor-866c9df647-kc54f namespace: kserve-ci-e2e-test resourceVersion: "20908" uid: 328ec472-00f3-45f9-ab8b-9ae649b29ee8 kind: Event lastTimestamp: "2026-07-02T13:38:46Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:38:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:38:46Z" name: success-200-isvc-6c5e8-predictor-866c9df647-kc54f.18be7c9f5bd3e603 namespace: kserve-ci-e2e-test resourceVersion: "20992" uid: f19a1bab-e0c9-4d51-94f4-c4933c286593 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:38:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6c5e8-predictor-866c9df647-kc54f namespace: kserve-ci-e2e-test resourceVersion: "20908" uid: 328ec472-00f3-45f9-ab8b-9ae649b29ee8 kind: Event lastTimestamp: "2026-07-02T13:39:22Z" message: 'Readiness probe failed: dial tcp 10.132.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:38:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:39:22Z" name: success-200-isvc-6c5e8-predictor-866c9df647-kc54f.18be7c9f5cab039a namespace: kserve-ci-e2e-test resourceVersion: "21283" uid: afdec10e-989f-4f35-aca4-dad43bd7caa7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6c5e8-predictor-866c9df647-kc54f namespace: kserve-ci-e2e-test resourceVersion: "20908" uid: 328ec472-00f3-45f9-ab8b-9ae649b29ee8 kind: Event lastTimestamp: "2026-07-02T13:39:55Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T13:39:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:39:55Z" name: success-200-isvc-6c5e8-predictor-866c9df647-kc54f.18be7caf706a44c4 namespace: kserve-ci-e2e-test resourceVersion: "21818" uid: 867a6ad1-5ee5-4532-bf38-09b7a3247db7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6c5e8-predictor-866c9df647-kc54f namespace: kserve-ci-e2e-test resourceVersion: "20908" uid: 328ec472-00f3-45f9-ab8b-9ae649b29ee8 kind: Event lastTimestamp: "2026-07-02T13:39:55Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:39:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:39:55Z" name: success-200-isvc-6c5e8-predictor-866c9df647-kc54f.18be7caf706abd7a namespace: kserve-ci-e2e-test resourceVersion: "21823" uid: fa487da2-1335-4efe-a79e-79cab780f7f4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6c5e8-predictor-866c9df647-kc54f namespace: kserve-ci-e2e-test resourceVersion: "20908" uid: 328ec472-00f3-45f9-ab8b-9ae649b29ee8 kind: Event lastTimestamp: "2026-07-02T13:39:57Z" message: 'Readiness probe failed: Get "https://10.132.0.37:8643/healthz": dial tcp 10.132.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:39:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:39:57Z" name: success-200-isvc-6c5e8-predictor-866c9df647-kc54f.18be7cafe4c8f010 namespace: kserve-ci-e2e-test resourceVersion: "21909" uid: e5c295ab-75a5-4c8b-a7b4-b4e5b5a9740c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:38:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-6c5e8-predictor-866c9df647 namespace: kserve-ci-e2e-test resourceVersion: "20890" uid: 560a7400-eb71-4718-a5bc-a090c33ede78 kind: Event lastTimestamp: "2026-07-02T13:38:45Z" message: 'Created pod: success-200-isvc-6c5e8-predictor-866c9df647-kc54f' metadata: creationTimestamp: "2026-07-02T13:38:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:38:45Z" name: success-200-isvc-6c5e8-predictor-866c9df647.18be7c9f14c38592 namespace: kserve-ci-e2e-test resourceVersion: "20907" uid: b57a360e-553d-45c8-b0c2-3276c949b152 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:38:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-6c5e8-predictor namespace: kserve-ci-e2e-test resourceVersion: "20882" uid: 10ae72d1-ab69-4400-b269-8855a7fece71 kind: Event lastTimestamp: "2026-07-02T13:38:45Z" message: Scaled up replica set success-200-isvc-6c5e8-predictor-866c9df647 from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:38:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:38:45Z" name: success-200-isvc-6c5e8-predictor.18be7c9f116eb83e namespace: kserve-ci-e2e-test resourceVersion: "20892" uid: a90b7d1c-3103-4bcb-b526-ff47a5d0ce1c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:39:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6c5e8-predictor namespace: kserve-ci-e2e-test resourceVersion: "20886" uid: c8577062-1619-4394-8557-a4b2a50c828c kind: Event lastTimestamp: "2026-07-02T13:39:15Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:39:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:39:15Z" name: success-200-isvc-6c5e8-predictor.18be7ca28fd2c27e namespace: kserve-ci-e2e-test resourceVersion: "21226" uid: fb34ba10-54ce-48cf-b2a7-68b13b32bc2b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:39:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6c5e8-predictor namespace: kserve-ci-e2e-test resourceVersion: "20886" uid: c8577062-1619-4394-8557-a4b2a50c828c kind: Event lastTimestamp: "2026-07-02T13:39:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:39:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:39:15Z" name: success-200-isvc-6c5e8-predictor.18be7ca290298f17 namespace: kserve-ci-e2e-test resourceVersion: "21227" uid: 2cfe3988-bd2e-4281-bb01-c371661834d1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:39:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6c5e8-predictor namespace: kserve-ci-e2e-test resourceVersion: "21115" uid: c8577062-1619-4394-8557-a4b2a50c828c kind: Event lastTimestamp: "2026-07-02T13:39:45Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:39:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:39:45Z" name: success-200-isvc-6c5e8-predictor.18be7ca98dc1b479 namespace: kserve-ci-e2e-test resourceVersion: "21635" uid: 19166996-5942-419e-978f-6fa137e277c5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:39:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6c5e8-predictor namespace: kserve-ci-e2e-test resourceVersion: "21115" uid: c8577062-1619-4394-8557-a4b2a50c828c kind: Event lastTimestamp: "2026-07-02T13:39:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:39:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:39:45Z" name: success-200-isvc-6c5e8-predictor.18be7ca98e20d345 namespace: kserve-ci-e2e-test resourceVersion: "21636" uid: 4ae9717c-6e17-4a13-ac19-c30d96b8ae12 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:38:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6c5e8 namespace: kserve-ci-e2e-test resourceVersion: "20870" uid: f1dc930f-b79a-4c43-81a7-755280f108fb kind: Event lastTimestamp: "2026-07-02T13:39:32Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T13:38:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:39:32Z" name: success-200-isvc-6c5e8.18be7c9f117e8bb8 namespace: kserve-ci-e2e-test resourceVersion: "21519" uid: 558aeb0c-2d46-417b-adda-320acb09807e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:38:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6c5e8 namespace: kserve-ci-e2e-test resourceVersion: "20870" uid: f1dc930f-b79a-4c43-81a7-755280f108fb kind: Event lastTimestamp: "2026-07-02T13:38:45Z" message: 'Failed to update status for InferenceService "success-200-isvc-6c5e8": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-6c5e8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:38:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:38:45Z" name: success-200-isvc-6c5e8.18be7c9f1204f0e5 namespace: kserve-ci-e2e-test resourceVersion: "20896" uid: ab110d78-ddfa-4d8d-ac49-416c1f8eb3e4 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:38:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6c5e8 namespace: kserve-ci-e2e-test resourceVersion: "20870" uid: f1dc930f-b79a-4c43-81a7-755280f108fb kind: Event lastTimestamp: "2026-07-02T13:38:45Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-6c5e8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:38:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:38:45Z" name: success-200-isvc-6c5e8.18be7c9f1205465f namespace: kserve-ci-e2e-test resourceVersion: "20898" uid: 9562afa6-48d8-47a3-bb7a-e7aef6cd3c90 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6c5e8 namespace: kserve-ci-e2e-test resourceVersion: "21517" uid: f1dc930f-b79a-4c43-81a7-755280f108fb kind: Event lastTimestamp: "2026-07-02T13:39:32Z" message: InferenceService [success-200-isvc-6c5e8] is Ready metadata: creationTimestamp: "2026-07-02T13:39:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:39:32Z" name: success-200-isvc-6c5e8.18be7caa1855e802 namespace: kserve-ci-e2e-test resourceVersion: "21518" uid: ca434b29-d838-4e29-92e0-981071727ce2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:40:35.120903Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7b15c-predictor-6cdf8ff9bf-rrrzc namespace: kserve-ci-e2e-test resourceVersion: "22324" uid: f1a08288-183d-4a3d-8202-cd0b559e5a05 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-7b15c-predictor-6cdf8ff9bf-rrrzc to ip-10-0-131-238.ec2.internal metadata: creationTimestamp: "2026-07-02T13:40:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:40:35Z" name: success-200-isvc-7b15c-predictor-6cdf8ff9bf-rrrzc.18be7cb8ae086e1e namespace: kserve-ci-e2e-test resourceVersion: "22336" uid: 6cbce2a4-8e47-428d-867a-08c64a1cb4fd reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7f4bb8bc8-mnq2w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:40:35Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7b15c-predictor-6cdf8ff9bf-rrrzc namespace: kserve-ci-e2e-test resourceVersion: "22344" uid: f1a08288-183d-4a3d-8202-cd0b559e5a05 kind: Event lastTimestamp: "2026-07-02T13:40:35Z" message: Add eth0 [10.132.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:40:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:40:35Z" name: success-200-isvc-7b15c-predictor-6cdf8ff9bf-rrrzc.18be7cb8c87d63ec namespace: kserve-ci-e2e-test resourceVersion: "22384" uid: 9389f3c2-835f-48e9-a6c1-fdfd0321b51b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:40:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7b15c-predictor-6cdf8ff9bf-rrrzc namespace: kserve-ci-e2e-test resourceVersion: "22330" uid: f1a08288-183d-4a3d-8202-cd0b559e5a05 kind: Event lastTimestamp: "2026-07-02T13:40:35Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1683" already present on machine metadata: creationTimestamp: "2026-07-02T13:40:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:40:35Z" name: success-200-isvc-7b15c-predictor-6cdf8ff9bf-rrrzc.18be7cb8c9a63c4c namespace: kserve-ci-e2e-test resourceVersion: "22387" uid: 698b8c62-aff1-4f02-a1a9-8468f9da5b09 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:40:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7b15c-predictor-6cdf8ff9bf-rrrzc namespace: kserve-ci-e2e-test resourceVersion: "22330" uid: f1a08288-183d-4a3d-8202-cd0b559e5a05 kind: Event lastTimestamp: "2026-07-02T13:40:35Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T13:40:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:40:35Z" name: success-200-isvc-7b15c-predictor-6cdf8ff9bf-rrrzc.18be7cb8ccb84989 namespace: kserve-ci-e2e-test resourceVersion: "22388" uid: 0988ffc9-2904-45d0-ac08-15be451041d7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:40:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7b15c-predictor-6cdf8ff9bf-rrrzc namespace: kserve-ci-e2e-test resourceVersion: "22330" uid: f1a08288-183d-4a3d-8202-cd0b559e5a05 kind: Event lastTimestamp: "2026-07-02T13:40:35Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T13:40:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:40:35Z" name: success-200-isvc-7b15c-predictor-6cdf8ff9bf-rrrzc.18be7cb8cd46d21c namespace: kserve-ci-e2e-test resourceVersion: "22390" uid: b13780d7-ca36-4aeb-91cd-374345c3aec4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:40:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7b15c-predictor-6cdf8ff9bf-rrrzc namespace: kserve-ci-e2e-test resourceVersion: "22330" uid: f1a08288-183d-4a3d-8202-cd0b559e5a05 kind: Event lastTimestamp: "2026-07-02T13:40:35Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T13:40:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:40:35Z" name: success-200-isvc-7b15c-predictor-6cdf8ff9bf-rrrzc.18be7cb8cd527468 namespace: kserve-ci-e2e-test resourceVersion: "22393" uid: 2d1cfb85-0de7-4896-8943-e1794af6ef0d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:40:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7b15c-predictor-6cdf8ff9bf-rrrzc namespace: kserve-ci-e2e-test resourceVersion: "22330" uid: f1a08288-183d-4a3d-8202-cd0b559e5a05 kind: Event lastTimestamp: "2026-07-02T13:40:35Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T13:40:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:40:35Z" name: success-200-isvc-7b15c-predictor-6cdf8ff9bf-rrrzc.18be7cb8cffb72ed namespace: kserve-ci-e2e-test resourceVersion: "22395" uid: 8d66f1f8-bb02-4395-8c28-ba2ed7fc1f28 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:40:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7b15c-predictor-6cdf8ff9bf-rrrzc namespace: kserve-ci-e2e-test resourceVersion: "22330" uid: f1a08288-183d-4a3d-8202-cd0b559e5a05 kind: Event lastTimestamp: "2026-07-02T13:40:35Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:40:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:40:35Z" name: success-200-isvc-7b15c-predictor-6cdf8ff9bf-rrrzc.18be7cb8d069dba2 namespace: kserve-ci-e2e-test resourceVersion: "22396" uid: cc03a967-24d4-4f12-ae19-2c3cb7642850 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-02T13:40:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7b15c-predictor-6cdf8ff9bf-rrrzc namespace: kserve-ci-e2e-test resourceVersion: "22330" uid: f1a08288-183d-4a3d-8202-cd0b559e5a05 kind: Event lastTimestamp: "2026-07-02T13:49:52Z" message: 'Readiness probe failed: dial tcp 10.132.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:40:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:49:52Z" name: success-200-isvc-7b15c-predictor-6cdf8ff9bf-rrrzc.18be7cb919b144f9 namespace: kserve-ci-e2e-test resourceVersion: "27010" uid: fa1df13a-5b59-4a31-b7cb-ad264a7f10e8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:49:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7b15c-predictor-6cdf8ff9bf-rrrzc namespace: kserve-ci-e2e-test resourceVersion: "22330" uid: f1a08288-183d-4a3d-8202-cd0b559e5a05 kind: Event lastTimestamp: "2026-07-02T13:49:50Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T13:49:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:49:50Z" name: success-200-isvc-7b15c-predictor-6cdf8ff9bf-rrrzc.18be7d39e3ea593b namespace: kserve-ci-e2e-test resourceVersion: "26912" uid: e9bb0e3c-bece-4d42-853e-e1b12e53591a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:49:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7b15c-predictor-6cdf8ff9bf-rrrzc namespace: kserve-ci-e2e-test resourceVersion: "22330" uid: f1a08288-183d-4a3d-8202-cd0b559e5a05 kind: Event lastTimestamp: "2026-07-02T13:49:50Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:49:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:49:50Z" name: success-200-isvc-7b15c-predictor-6cdf8ff9bf-rrrzc.18be7d39e3eae8bf namespace: kserve-ci-e2e-test resourceVersion: "26913" uid: e1beef6b-1e60-4a14-ba32-669552fdf2c8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:49:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7b15c-predictor-6cdf8ff9bf-rrrzc namespace: kserve-ci-e2e-test resourceVersion: "22330" uid: f1a08288-183d-4a3d-8202-cd0b559e5a05 kind: Event lastTimestamp: "2026-07-02T13:49:52Z" message: 'Readiness probe failed: Get "https://10.132.0.40:8643/healthz": dial tcp 10.132.0.40:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:49:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:49:52Z" name: success-200-isvc-7b15c-predictor-6cdf8ff9bf-rrrzc.18be7d3a8e1dc6cc namespace: kserve-ci-e2e-test resourceVersion: "27009" uid: 6175d71a-8a09-498b-a77b-54ac4fb200c2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:40:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-7b15c-predictor-6cdf8ff9bf namespace: kserve-ci-e2e-test resourceVersion: "22308" uid: b3e791d0-9185-474c-b66f-0daee2a3b2e1 kind: Event lastTimestamp: "2026-07-02T13:40:35Z" message: 'Created pod: success-200-isvc-7b15c-predictor-6cdf8ff9bf-rrrzc' metadata: creationTimestamp: "2026-07-02T13:40:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:40:35Z" name: success-200-isvc-7b15c-predictor-6cdf8ff9bf.18be7cb8ad793c4c namespace: kserve-ci-e2e-test resourceVersion: "22329" uid: b96a198a-72e8-4fe7-86fe-d93034827d7a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:40:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-7b15c-predictor namespace: kserve-ci-e2e-test resourceVersion: "22305" uid: fcd33060-9b65-41c6-a337-32521a462e6f kind: Event lastTimestamp: "2026-07-02T13:40:35Z" message: Scaled up replica set success-200-isvc-7b15c-predictor-6cdf8ff9bf from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:40:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:40:35Z" name: success-200-isvc-7b15c-predictor.18be7cb8ab9db4f8 namespace: kserve-ci-e2e-test resourceVersion: "22309" uid: c8116eae-9f05-439c-8cfb-0763c0a6d9f1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:40:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7b15c-predictor namespace: kserve-ci-e2e-test resourceVersion: "22307" uid: 0f8f0234-107c-462d-94b2-f2c36c916849 kind: Event lastTimestamp: "2026-07-02T13:41:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:40:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:41:20Z" name: success-200-isvc-7b15c-predictor.18be7cbc2a125607 namespace: kserve-ci-e2e-test resourceVersion: "22830" uid: 068b6221-664f-4dcc-8066-73c21408cca6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:40:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7b15c-predictor namespace: kserve-ci-e2e-test resourceVersion: "22307" uid: 0f8f0234-107c-462d-94b2-f2c36c916849 kind: Event lastTimestamp: "2026-07-02T13:41:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:40:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:41:20Z" name: success-200-isvc-7b15c-predictor.18be7cbc2a72a4dc namespace: kserve-ci-e2e-test resourceVersion: "22831" uid: 6e132e69-6182-4027-a1c8-78e1fa7b6b7f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:41:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7b15c-predictor namespace: kserve-ci-e2e-test resourceVersion: "22553" uid: 0f8f0234-107c-462d-94b2-f2c36c916849 kind: Event lastTimestamp: "2026-07-02T13:41:50Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:41:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:41:50Z" name: success-200-isvc-7b15c-predictor.18be7cbfa8f91312 namespace: kserve-ci-e2e-test resourceVersion: "23101" uid: 2947487c-991f-4775-960b-cff16e9a370d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:41:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7b15c-predictor namespace: kserve-ci-e2e-test resourceVersion: "22553" uid: 0f8f0234-107c-462d-94b2-f2c36c916849 kind: Event lastTimestamp: "2026-07-02T13:41:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:41:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:41:50Z" name: success-200-isvc-7b15c-predictor.18be7cbfa957283d namespace: kserve-ci-e2e-test resourceVersion: "23102" uid: 235d2308-458d-400d-a062-40a82d69f6b3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T13:40:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7b15c namespace: kserve-ci-e2e-test resourceVersion: "22288" uid: dd94c640-f8e0-4e8d-afce-3ee9dab219f9 kind: Event lastTimestamp: "2026-07-02T13:41:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T13:40:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:41:23Z" name: success-200-isvc-7b15c.18be7cb8aba4d7f6 namespace: kserve-ci-e2e-test resourceVersion: "22857" uid: 4341c685-0c59-4756-9450-a7636b8930e5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:41:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7b15c namespace: kserve-ci-e2e-test resourceVersion: "22855" uid: dd94c640-f8e0-4e8d-afce-3ee9dab219f9 kind: Event lastTimestamp: "2026-07-02T13:41:23Z" message: InferenceService [success-200-isvc-7b15c] is Ready metadata: creationTimestamp: "2026-07-02T13:41:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:41:23Z" name: success-200-isvc-7b15c.18be7cc3d404238d namespace: kserve-ci-e2e-test resourceVersion: "22856" uid: f0e281c8-1e36-400c-afeb-b851483f5d1f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:49:50.097855Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7f184-predictor-5f7fcfbcbf-747l7 namespace: kserve-ci-e2e-test resourceVersion: "26915" uid: 1f204ab2-93ae-427e-ac6d-d436baa331e9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-7f184-predictor-5f7fcfbcbf-747l7 to ip-10-0-131-238.ec2.internal metadata: creationTimestamp: "2026-07-02T13:49:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:49:50Z" name: success-200-isvc-7f184-predictor-5f7fcfbcbf-747l7.18be7d39e53cb72e namespace: kserve-ci-e2e-test resourceVersion: "26920" uid: f4557d36-20fd-4b71-a7f9-1035d2a256bf reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7f4bb8bc8-mnq2w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:49:50Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7f184-predictor-5f7fcfbcbf-747l7 namespace: kserve-ci-e2e-test resourceVersion: "26917" uid: 1f204ab2-93ae-427e-ac6d-d436baa331e9 kind: Event lastTimestamp: "2026-07-02T13:49:50Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-7f184-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T13:49:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:49:50Z" name: success-200-isvc-7f184-predictor-5f7fcfbcbf-747l7.18be7d39ef1b7776 namespace: kserve-ci-e2e-test resourceVersion: "26956" uid: 866a3e45-31ab-4780-9395-5e96dadb902c reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:49:51Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7f184-predictor-5f7fcfbcbf-747l7 namespace: kserve-ci-e2e-test resourceVersion: "26932" uid: 1f204ab2-93ae-427e-ac6d-d436baa331e9 kind: Event lastTimestamp: "2026-07-02T13:49:51Z" message: Add eth0 [10.132.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:49:51Z" name: success-200-isvc-7f184-predictor-5f7fcfbcbf-747l7.18be7d3a22b2bbd0 namespace: kserve-ci-e2e-test resourceVersion: "26970" uid: 13e03070-84ec-4b0b-bb4c-7d6fed081f9c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:49:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7f184-predictor-5f7fcfbcbf-747l7 namespace: kserve-ci-e2e-test resourceVersion: "26917" uid: 1f204ab2-93ae-427e-ac6d-d436baa331e9 kind: Event lastTimestamp: "2026-07-02T13:49:51Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1683" already present on machine metadata: creationTimestamp: "2026-07-02T13:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:49:51Z" name: success-200-isvc-7f184-predictor-5f7fcfbcbf-747l7.18be7d3a2437ffec namespace: kserve-ci-e2e-test resourceVersion: "26974" uid: 6c1090b0-cba5-465b-98ee-a2d71461ce2c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:49:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7f184-predictor-5f7fcfbcbf-747l7 namespace: kserve-ci-e2e-test resourceVersion: "26917" uid: 1f204ab2-93ae-427e-ac6d-d436baa331e9 kind: Event lastTimestamp: "2026-07-02T13:49:51Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T13:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:49:51Z" name: success-200-isvc-7f184-predictor-5f7fcfbcbf-747l7.18be7d3a27715058 namespace: kserve-ci-e2e-test resourceVersion: "26977" uid: 506d3008-814b-4859-aad0-3e14cec7a9e6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:49:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7f184-predictor-5f7fcfbcbf-747l7 namespace: kserve-ci-e2e-test resourceVersion: "26917" uid: 1f204ab2-93ae-427e-ac6d-d436baa331e9 kind: Event lastTimestamp: "2026-07-02T13:49:51Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T13:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:49:51Z" name: success-200-isvc-7f184-predictor-5f7fcfbcbf-747l7.18be7d3a27e4a191 namespace: kserve-ci-e2e-test resourceVersion: "26978" uid: f4ad2e4e-e1e9-43f8-a2cd-c3ddbfaed2eb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:49:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7f184-predictor-5f7fcfbcbf-747l7 namespace: kserve-ci-e2e-test resourceVersion: "26917" uid: 1f204ab2-93ae-427e-ac6d-d436baa331e9 kind: Event lastTimestamp: "2026-07-02T13:49:51Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T13:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:49:51Z" name: success-200-isvc-7f184-predictor-5f7fcfbcbf-747l7.18be7d3a27ef1ef7 namespace: kserve-ci-e2e-test resourceVersion: "26979" uid: 02b9b5c5-357c-469c-ace3-f660973b4345 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:49:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7f184-predictor-5f7fcfbcbf-747l7 namespace: kserve-ci-e2e-test resourceVersion: "26917" uid: 1f204ab2-93ae-427e-ac6d-d436baa331e9 kind: Event lastTimestamp: "2026-07-02T13:49:51Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T13:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:49:51Z" name: success-200-isvc-7f184-predictor-5f7fcfbcbf-747l7.18be7d3a2b41246d namespace: kserve-ci-e2e-test resourceVersion: "26982" uid: 554ae6a6-ec3a-4c96-9e66-d1ecec633f75 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:49:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7f184-predictor-5f7fcfbcbf-747l7 namespace: kserve-ci-e2e-test resourceVersion: "26917" uid: 1f204ab2-93ae-427e-ac6d-d436baa331e9 kind: Event lastTimestamp: "2026-07-02T13:49:51Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:49:51Z" name: success-200-isvc-7f184-predictor-5f7fcfbcbf-747l7.18be7d3a2bb497eb namespace: kserve-ci-e2e-test resourceVersion: "26984" uid: 5fe300e8-5e2f-4e83-9550-7ab72cdae1bf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:49:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7f184-predictor-5f7fcfbcbf-747l7 namespace: kserve-ci-e2e-test resourceVersion: "26917" uid: 1f204ab2-93ae-427e-ac6d-d436baa331e9 kind: Event lastTimestamp: "2026-07-02T13:50:28Z" message: 'Readiness probe failed: dial tcp 10.132.0.42:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:49:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:50:28Z" name: success-200-isvc-7f184-predictor-5f7fcfbcbf-747l7.18be7d3a78cb2ab8 namespace: kserve-ci-e2e-test resourceVersion: "27498" uid: 66ebc9fc-ec8f-48dd-aa1a-ea5bb4b36bf6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:07:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7f184-predictor-5f7fcfbcbf-747l7 namespace: kserve-ci-e2e-test resourceVersion: "26917" uid: 1f204ab2-93ae-427e-ac6d-d436baa331e9 kind: Event lastTimestamp: "2026-07-02T14:07:09Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:07:09Z" name: success-200-isvc-7f184-predictor-5f7fcfbcbf-747l7.18be7e2bebab035a namespace: kserve-ci-e2e-test resourceVersion: "34646" uid: 3e6f873c-6d16-440a-ba20-0472bb798a86 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:07:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7f184-predictor-5f7fcfbcbf-747l7 namespace: kserve-ci-e2e-test resourceVersion: "26917" uid: 1f204ab2-93ae-427e-ac6d-d436baa331e9 kind: Event lastTimestamp: "2026-07-02T14:07:09Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:07:09Z" name: success-200-isvc-7f184-predictor-5f7fcfbcbf-747l7.18be7e2bebac04be namespace: kserve-ci-e2e-test resourceVersion: "34647" uid: f29f6d3b-ab3c-46d3-a6e1-13acd9cd2a84 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:49:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-7f184-predictor-5f7fcfbcbf namespace: kserve-ci-e2e-test resourceVersion: "26901" uid: 75556a23-caa3-4ccb-b98b-92613ed748fd kind: Event lastTimestamp: "2026-07-02T13:49:50Z" message: 'Created pod: success-200-isvc-7f184-predictor-5f7fcfbcbf-747l7' metadata: creationTimestamp: "2026-07-02T13:49:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:49:50Z" name: success-200-isvc-7f184-predictor-5f7fcfbcbf.18be7d39e4dc408c namespace: kserve-ci-e2e-test resourceVersion: "26918" uid: 690d70b6-22e0-4d64-a9d9-f6241f8f2a1a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:49:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-7f184-predictor namespace: kserve-ci-e2e-test resourceVersion: "26893" uid: e34f3120-e302-4a98-bcb6-bdda764e87d9 kind: Event lastTimestamp: "2026-07-02T13:49:50Z" message: Scaled up replica set success-200-isvc-7f184-predictor-5f7fcfbcbf from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:49:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:49:50Z" name: success-200-isvc-7f184-predictor.18be7d39e2da6bab namespace: kserve-ci-e2e-test resourceVersion: "26906" uid: 16f633e6-d474-410d-8489-06d2b2866e9b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:50:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7f184-predictor namespace: kserve-ci-e2e-test resourceVersion: "26899" uid: df821629-77f3-4e6f-9bcc-c50472e4350d kind: Event lastTimestamp: "2026-07-02T13:50:35Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:50:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:50:35Z" name: success-200-isvc-7f184-predictor.18be7d3d611f9bab namespace: kserve-ci-e2e-test resourceVersion: "27557" uid: 31f5fd49-58e7-4703-870d-63b53d87d536 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:50:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7f184-predictor namespace: kserve-ci-e2e-test resourceVersion: "26899" uid: df821629-77f3-4e6f-9bcc-c50472e4350d kind: Event lastTimestamp: "2026-07-02T13:50:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:50:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:50:35Z" name: success-200-isvc-7f184-predictor.18be7d3d617d3e25 namespace: kserve-ci-e2e-test resourceVersion: "27558" uid: c870523d-4e7c-4229-9fb9-50d07850e743 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:50:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7f184-predictor namespace: kserve-ci-e2e-test resourceVersion: "27131" uid: df821629-77f3-4e6f-9bcc-c50472e4350d kind: Event lastTimestamp: "2026-07-02T13:51:05Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:50:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:51:05Z" name: success-200-isvc-7f184-predictor.18be7d40e007ab2e namespace: kserve-ci-e2e-test resourceVersion: "27861" uid: d5f88747-851c-4c67-8f0d-a280f38c4d4c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:50:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7f184-predictor namespace: kserve-ci-e2e-test resourceVersion: "27131" uid: df821629-77f3-4e6f-9bcc-c50472e4350d kind: Event lastTimestamp: "2026-07-02T13:51:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:50:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:51:05Z" name: success-200-isvc-7f184-predictor.18be7d40e063b6ef namespace: kserve-ci-e2e-test resourceVersion: "27862" uid: 2dda8012-46ed-4cd8-b623-79ae3e3e9ea9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:49:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7f184 namespace: kserve-ci-e2e-test resourceVersion: "26869" uid: f4dd2394-e514-4da9-bace-a23bba7a43b1 kind: Event lastTimestamp: "2026-07-02T13:50:38Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T13:49:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:50:38Z" name: success-200-isvc-7f184.18be7d39e2cf7a6c namespace: kserve-ci-e2e-test resourceVersion: "27599" uid: 98793a19-8411-40fd-988c-5d59b3ebd944 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:49:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7f184 namespace: kserve-ci-e2e-test resourceVersion: "26869" uid: f4dd2394-e514-4da9-bace-a23bba7a43b1 kind: Event lastTimestamp: "2026-07-02T13:49:50Z" message: 'Failed to update status for InferenceService "success-200-isvc-7f184": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-7f184": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:49:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:49:50Z" name: success-200-isvc-7f184.18be7d39e3628750 namespace: kserve-ci-e2e-test resourceVersion: "26909" uid: c79e09b0-05e6-4178-be69-5d3314759662 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:49:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7f184 namespace: kserve-ci-e2e-test resourceVersion: "26869" uid: f4dd2394-e514-4da9-bace-a23bba7a43b1 kind: Event lastTimestamp: "2026-07-02T13:49:50Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-7f184": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:49:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:49:50Z" name: success-200-isvc-7f184.18be7d39e362d1f2 namespace: kserve-ci-e2e-test resourceVersion: "26910" uid: 847a89e9-0b0f-4aa4-8107-a391c24fc25e reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:50:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7f184 namespace: kserve-ci-e2e-test resourceVersion: "27596" uid: f4dd2394-e514-4da9-bace-a23bba7a43b1 kind: Event lastTimestamp: "2026-07-02T13:50:38Z" message: InferenceService [success-200-isvc-7f184] is Ready metadata: creationTimestamp: "2026-07-02T13:50:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:50:38Z" name: success-200-isvc-7f184.18be7d45333c2f2c namespace: kserve-ci-e2e-test resourceVersion: "27598" uid: 8b41c7cf-99a7-4ca0-ab3d-0c64f09df648 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:49:15.413298Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-9633b-predictor-774c89fdf5-kzkcr namespace: kserve-ci-e2e-test resourceVersion: "26476" uid: 27261a17-2625-4494-bd67-c89c19d41f32 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-9633b-predictor-774c89fdf5-kzkcr to ip-10-0-131-238.ec2.internal metadata: creationTimestamp: "2026-07-02T13:49:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:49:15Z" name: success-200-isvc-9633b-predictor-774c89fdf5-kzkcr.18be7d31d1e06201 namespace: kserve-ci-e2e-test resourceVersion: "26492" uid: 67799432-857f-43ac-b833-d063cf9b1059 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7f4bb8bc8-mnq2w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:49:15Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-9633b-predictor-774c89fdf5-kzkcr namespace: kserve-ci-e2e-test resourceVersion: "26504" uid: 27261a17-2625-4494-bd67-c89c19d41f32 kind: Event lastTimestamp: "2026-07-02T13:49:15Z" message: Add eth0 [10.132.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:49:15Z" name: success-200-isvc-9633b-predictor-774c89fdf5-kzkcr.18be7d31ec6f8d72 namespace: kserve-ci-e2e-test resourceVersion: "26554" uid: 4041c9fe-7c2c-46f8-aae5-ed637b48015a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:49:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9633b-predictor-774c89fdf5-kzkcr namespace: kserve-ci-e2e-test resourceVersion: "26484" uid: 27261a17-2625-4494-bd67-c89c19d41f32 kind: Event lastTimestamp: "2026-07-02T13:49:15Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1683" already present on machine metadata: creationTimestamp: "2026-07-02T13:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:49:15Z" name: success-200-isvc-9633b-predictor-774c89fdf5-kzkcr.18be7d31edb89456 namespace: kserve-ci-e2e-test resourceVersion: "26556" uid: f2765c04-ad30-461a-ab51-4c508a791c0d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:49:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9633b-predictor-774c89fdf5-kzkcr namespace: kserve-ci-e2e-test resourceVersion: "26484" uid: 27261a17-2625-4494-bd67-c89c19d41f32 kind: Event lastTimestamp: "2026-07-02T13:49:15Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T13:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:49:15Z" name: success-200-isvc-9633b-predictor-774c89fdf5-kzkcr.18be7d31f07a9366 namespace: kserve-ci-e2e-test resourceVersion: "26559" uid: 9b9a1c95-85f0-430c-9e2f-4e0af1c61a15 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:49:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9633b-predictor-774c89fdf5-kzkcr namespace: kserve-ci-e2e-test resourceVersion: "26484" uid: 27261a17-2625-4494-bd67-c89c19d41f32 kind: Event lastTimestamp: "2026-07-02T13:49:15Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T13:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:49:15Z" name: success-200-isvc-9633b-predictor-774c89fdf5-kzkcr.18be7d31f0f24f61 namespace: kserve-ci-e2e-test resourceVersion: "26561" uid: d3a88c93-0013-46b5-88c4-f2926e741533 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:49:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9633b-predictor-774c89fdf5-kzkcr namespace: kserve-ci-e2e-test resourceVersion: "26484" uid: 27261a17-2625-4494-bd67-c89c19d41f32 kind: Event lastTimestamp: "2026-07-02T13:49:15Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T13:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:49:15Z" name: success-200-isvc-9633b-predictor-774c89fdf5-kzkcr.18be7d31f0fcde5a namespace: kserve-ci-e2e-test resourceVersion: "26563" uid: 87a7d5ea-2069-401b-a297-14ffa1fecc59 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:49:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9633b-predictor-774c89fdf5-kzkcr namespace: kserve-ci-e2e-test resourceVersion: "26484" uid: 27261a17-2625-4494-bd67-c89c19d41f32 kind: Event lastTimestamp: "2026-07-02T13:49:15Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T13:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:49:15Z" name: success-200-isvc-9633b-predictor-774c89fdf5-kzkcr.18be7d31f3e88cc9 namespace: kserve-ci-e2e-test resourceVersion: "26566" uid: 7f60fbea-95df-46e4-8f92-3bcf8d2f1cf8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:49:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9633b-predictor-774c89fdf5-kzkcr namespace: kserve-ci-e2e-test resourceVersion: "26484" uid: 27261a17-2625-4494-bd67-c89c19d41f32 kind: Event lastTimestamp: "2026-07-02T13:49:15Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:49:15Z" name: success-200-isvc-9633b-predictor-774c89fdf5-kzkcr.18be7d31f45a453a namespace: kserve-ci-e2e-test resourceVersion: "26568" uid: f2a03e94-6239-4b10-a50e-fe1a75e767dd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:49:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9633b-predictor-774c89fdf5-kzkcr namespace: kserve-ci-e2e-test resourceVersion: "26484" uid: 27261a17-2625-4494-bd67-c89c19d41f32 kind: Event lastTimestamp: "2026-07-02T13:49:53Z" message: 'Readiness probe failed: dial tcp 10.132.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:49:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:49:53Z" name: success-200-isvc-9633b-predictor-774c89fdf5-kzkcr.18be7d3247ff806e namespace: kserve-ci-e2e-test resourceVersion: "27014" uid: b87e579a-3f71-4b52-9a0d-f326df4e6975 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:50:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9633b-predictor-774c89fdf5-kzkcr namespace: kserve-ci-e2e-test resourceVersion: "26484" uid: 27261a17-2625-4494-bd67-c89c19d41f32 kind: Event lastTimestamp: "2026-07-02T13:50:25Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T13:50:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:50:25Z" name: success-200-isvc-9633b-predictor-774c89fdf5-kzkcr.18be7d422c8d319f namespace: kserve-ci-e2e-test resourceVersion: "27391" uid: 1a3fec8f-c2af-4d20-a44c-dc5a15c3d860 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:50:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9633b-predictor-774c89fdf5-kzkcr namespace: kserve-ci-e2e-test resourceVersion: "26484" uid: 27261a17-2625-4494-bd67-c89c19d41f32 kind: Event lastTimestamp: "2026-07-02T13:50:25Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:50:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:50:25Z" name: success-200-isvc-9633b-predictor-774c89fdf5-kzkcr.18be7d422c8e0956 namespace: kserve-ci-e2e-test resourceVersion: "27395" uid: 4b459542-c555-4708-9433-f52c51612e20 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:50:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9633b-predictor-774c89fdf5-kzkcr namespace: kserve-ci-e2e-test resourceVersion: "26484" uid: 27261a17-2625-4494-bd67-c89c19d41f32 kind: Event lastTimestamp: "2026-07-02T13:50:28Z" message: 'Readiness probe failed: Get "https://10.132.0.41:8643/healthz": dial tcp 10.132.0.41:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:50:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:50:28Z" name: success-200-isvc-9633b-predictor-774c89fdf5-kzkcr.18be7d42d02def74 namespace: kserve-ci-e2e-test resourceVersion: "27497" uid: 01b2fde6-dd64-4263-ae15-12e67ea514a5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:49:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-9633b-predictor-774c89fdf5 namespace: kserve-ci-e2e-test resourceVersion: "26467" uid: 233c2821-5e09-4d18-85bc-fa1b57db8edf kind: Event lastTimestamp: "2026-07-02T13:49:15Z" message: 'Created pod: success-200-isvc-9633b-predictor-774c89fdf5-kzkcr' metadata: creationTimestamp: "2026-07-02T13:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:49:15Z" name: success-200-isvc-9633b-predictor-774c89fdf5.18be7d31d166896b namespace: kserve-ci-e2e-test resourceVersion: "26480" uid: b7f0c7be-7625-409c-89c5-e4514b86308c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:49:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-9633b-predictor namespace: kserve-ci-e2e-test resourceVersion: "26464" uid: c282a1e9-5568-489d-947a-cf3f37368551 kind: Event lastTimestamp: "2026-07-02T13:49:15Z" message: Scaled up replica set success-200-isvc-9633b-predictor-774c89fdf5 from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:49:15Z" name: success-200-isvc-9633b-predictor.18be7d31ce4d2770 namespace: kserve-ci-e2e-test resourceVersion: "26469" uid: f7f58120-5771-4536-b7bd-80ee2882837d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:49:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9633b-predictor namespace: kserve-ci-e2e-test resourceVersion: "26465" uid: adb94fb0-c4f5-4311-91bf-6a9eb3c41aa0 kind: Event lastTimestamp: "2026-07-02T13:49:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:49:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:49:30Z" name: success-200-isvc-9633b-predictor.18be7d354ce19a26 namespace: kserve-ci-e2e-test resourceVersion: "26693" uid: 1c478783-eb81-44bf-aafa-0a272eae977d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:49:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9633b-predictor namespace: kserve-ci-e2e-test resourceVersion: "26465" uid: adb94fb0-c4f5-4311-91bf-6a9eb3c41aa0 kind: Event lastTimestamp: "2026-07-02T13:49:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:49:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:49:30Z" name: success-200-isvc-9633b-predictor.18be7d354d4d45f3 namespace: kserve-ci-e2e-test resourceVersion: "26695" uid: f444ee88-df1a-4f74-8210-af31389913ed reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:49:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9633b-predictor namespace: kserve-ci-e2e-test resourceVersion: "26694" uid: adb94fb0-c4f5-4311-91bf-6a9eb3c41aa0 kind: Event lastTimestamp: "2026-07-02T13:50:15Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:49:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:50:15Z" name: success-200-isvc-9633b-predictor.18be7d38cbe6e50d namespace: kserve-ci-e2e-test resourceVersion: "27215" uid: 5bb0f446-eb6b-432f-a9dc-5f0e6983871c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:49:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9633b-predictor namespace: kserve-ci-e2e-test resourceVersion: "26694" uid: adb94fb0-c4f5-4311-91bf-6a9eb3c41aa0 kind: Event lastTimestamp: "2026-07-02T13:50:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:49:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:50:15Z" name: success-200-isvc-9633b-predictor.18be7d38cc468ca6 namespace: kserve-ci-e2e-test resourceVersion: "27216" uid: 5f73f43a-5854-4776-a208-09e94e30e2f3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T13:49:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9633b namespace: kserve-ci-e2e-test resourceVersion: "26454" uid: 3a2c71de-7cce-4076-9807-19615eddf729 kind: Event lastTimestamp: "2026-07-02T13:50:03Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T13:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:50:03Z" name: success-200-isvc-9633b.18be7d31ce47bbf0 namespace: kserve-ci-e2e-test resourceVersion: "27120" uid: cd54a044-1bdd-4782-a253-f8c8c5523eee reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:49:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9633b namespace: kserve-ci-e2e-test resourceVersion: "26454" uid: 3a2c71de-7cce-4076-9807-19615eddf729 kind: Event lastTimestamp: "2026-07-02T13:49:15Z" message: 'Failed to update status for InferenceService "success-200-isvc-9633b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-9633b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:49:15Z" name: success-200-isvc-9633b.18be7d31ce925f2a namespace: kserve-ci-e2e-test resourceVersion: "26471" uid: 6d2ef461-c1ca-4ed9-b332-00e2090626a0 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:49:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9633b namespace: kserve-ci-e2e-test resourceVersion: "26454" uid: 3a2c71de-7cce-4076-9807-19615eddf729 kind: Event lastTimestamp: "2026-07-02T13:49:15Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-9633b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:49:15Z" name: success-200-isvc-9633b.18be7d31ce92b5dc namespace: kserve-ci-e2e-test resourceVersion: "26472" uid: b43ad803-d4aa-4861-8915-80b7d5135e0b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:50:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9633b namespace: kserve-ci-e2e-test resourceVersion: "27117" uid: 3a2c71de-7cce-4076-9807-19615eddf729 kind: Event lastTimestamp: "2026-07-02T13:50:03Z" message: InferenceService [success-200-isvc-9633b] is Ready metadata: creationTimestamp: "2026-07-02T13:50:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:50:03Z" name: success-200-isvc-9633b.18be7d3d026a3c5e namespace: kserve-ci-e2e-test resourceVersion: "27119" uid: dcaf973f-cd71-4826-838a-eca52cd66e75 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:27:39.753894Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-acf44-predictor-76c47c8649-p5q4f namespace: kserve-ci-e2e-test resourceVersion: "15286" uid: 0c1ea2ad-39f7-425d-92c0-b44584cd5cd9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-acf44-predictor-76c47c8649-p5q4f to ip-10-0-131-238.ec2.internal metadata: creationTimestamp: "2026-07-02T13:27:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:27:39Z" name: success-200-isvc-acf44-predictor-76c47c8649-p5q4f.18be7c04268ed0b1 namespace: kserve-ci-e2e-test resourceVersion: "15291" uid: b1dc8b3a-55fd-4852-aa77-204dbfffc8d7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7f4bb8bc8-mnq2w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:27:40Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-acf44-predictor-76c47c8649-p5q4f namespace: kserve-ci-e2e-test resourceVersion: "15297" uid: 0c1ea2ad-39f7-425d-92c0-b44584cd5cd9 kind: Event lastTimestamp: "2026-07-02T13:27:40Z" message: Add eth0 [10.132.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:27:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:27:40Z" name: success-200-isvc-acf44-predictor-76c47c8649-p5q4f.18be7c04428554c3 namespace: kserve-ci-e2e-test resourceVersion: "15375" uid: 911e9ec7-b659-4f51-87d4-60ee6742716f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:27:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-acf44-predictor-76c47c8649-p5q4f namespace: kserve-ci-e2e-test resourceVersion: "15288" uid: 0c1ea2ad-39f7-425d-92c0-b44584cd5cd9 kind: Event lastTimestamp: "2026-07-02T13:27:40Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1683" metadata: creationTimestamp: "2026-07-02T13:27:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:27:40Z" name: success-200-isvc-acf44-predictor-76c47c8649-p5q4f.18be7c044434ec7b namespace: kserve-ci-e2e-test resourceVersion: "15383" uid: 3cec3b4c-2836-4341-83f7-1771199f1234 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:28:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-acf44-predictor-76c47c8649-p5q4f namespace: kserve-ci-e2e-test resourceVersion: "15288" uid: 0c1ea2ad-39f7-425d-92c0-b44584cd5cd9 kind: Event lastTimestamp: "2026-07-02T13:28:02Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1683" in 22.193s (22.194s including waiting). Image size: 1335613021 bytes.' metadata: creationTimestamp: "2026-07-02T13:28:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:28:02Z" name: success-200-isvc-acf44-predictor-76c47c8649-p5q4f.18be7c096f131197 namespace: kserve-ci-e2e-test resourceVersion: "15668" uid: 25b492b3-d5dc-4d8a-8785-64185b9e9d8f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:28:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-acf44-predictor-76c47c8649-p5q4f namespace: kserve-ci-e2e-test resourceVersion: "15288" uid: 0c1ea2ad-39f7-425d-92c0-b44584cd5cd9 kind: Event lastTimestamp: "2026-07-02T13:28:02Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T13:28:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:28:02Z" name: success-200-isvc-acf44-predictor-76c47c8649-p5q4f.18be7c0974761a08 namespace: kserve-ci-e2e-test resourceVersion: "15672" uid: a04c17d4-54c8-427b-b64e-966f7c1f1ed8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:28:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-acf44-predictor-76c47c8649-p5q4f namespace: kserve-ci-e2e-test resourceVersion: "15288" uid: 0c1ea2ad-39f7-425d-92c0-b44584cd5cd9 kind: Event lastTimestamp: "2026-07-02T13:28:02Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T13:28:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:28:02Z" name: success-200-isvc-acf44-predictor-76c47c8649-p5q4f.18be7c097507bc45 namespace: kserve-ci-e2e-test resourceVersion: "15673" uid: 63fc6cc1-d295-46c7-8982-60f240a1c5c5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:28:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-acf44-predictor-76c47c8649-p5q4f namespace: kserve-ci-e2e-test resourceVersion: "15288" uid: 0c1ea2ad-39f7-425d-92c0-b44584cd5cd9 kind: Event lastTimestamp: "2026-07-02T13:28:02Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-02T13:28:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:28:02Z" name: success-200-isvc-acf44-predictor-76c47c8649-p5q4f.18be7c09751fd529 namespace: kserve-ci-e2e-test resourceVersion: "15674" uid: 2e3862c5-8891-41a1-beb6-8a475428e878 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:28:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-acf44-predictor-76c47c8649-p5q4f namespace: kserve-ci-e2e-test resourceVersion: "15288" uid: 0c1ea2ad-39f7-425d-92c0-b44584cd5cd9 kind: Event lastTimestamp: "2026-07-02T13:28:06Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 4.124s (4.124s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-02T13:28:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:28:06Z" name: success-200-isvc-acf44-predictor-76c47c8649-p5q4f.18be7c0a6af7e5bb namespace: kserve-ci-e2e-test resourceVersion: "15712" uid: 132f63c3-f307-4915-b10b-7e83697ccf50 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:28:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-acf44-predictor-76c47c8649-p5q4f namespace: kserve-ci-e2e-test resourceVersion: "15288" uid: 0c1ea2ad-39f7-425d-92c0-b44584cd5cd9 kind: Event lastTimestamp: "2026-07-02T13:28:06Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T13:28:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:28:06Z" name: success-200-isvc-acf44-predictor-76c47c8649-p5q4f.18be7c0a6ee1975b namespace: kserve-ci-e2e-test resourceVersion: "15717" uid: f3e4fe13-3624-41cf-9c53-9fc372e0df5f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:28:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-acf44-predictor-76c47c8649-p5q4f namespace: kserve-ci-e2e-test resourceVersion: "15288" uid: 0c1ea2ad-39f7-425d-92c0-b44584cd5cd9 kind: Event lastTimestamp: "2026-07-02T13:28:06Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:28:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:28:06Z" name: success-200-isvc-acf44-predictor-76c47c8649-p5q4f.18be7c0a6faa506e namespace: kserve-ci-e2e-test resourceVersion: "15718" uid: 260123fd-0fce-4f72-805a-01f9365f964e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-02T13:28:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-acf44-predictor-76c47c8649-p5q4f namespace: kserve-ci-e2e-test resourceVersion: "15288" uid: 0c1ea2ad-39f7-425d-92c0-b44584cd5cd9 kind: Event lastTimestamp: "2026-07-02T13:29:33Z" message: 'Readiness probe failed: dial tcp 10.132.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:28:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:29:33Z" name: success-200-isvc-acf44-predictor-76c47c8649-p5q4f.18be7c0a8f863670 namespace: kserve-ci-e2e-test resourceVersion: "16663" uid: edee2f41-97af-49d3-8068-164d5d2d521e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:29:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-acf44-predictor-76c47c8649-p5q4f namespace: kserve-ci-e2e-test resourceVersion: "15288" uid: 0c1ea2ad-39f7-425d-92c0-b44584cd5cd9 kind: Event lastTimestamp: "2026-07-02T13:29:30Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T13:29:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:29:30Z" name: success-200-isvc-acf44-predictor-76c47c8649-p5q4f.18be7c1ddff41811 namespace: kserve-ci-e2e-test resourceVersion: "16546" uid: f09754e4-b520-4b30-8970-4199950e9089 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:29:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-acf44-predictor-76c47c8649-p5q4f namespace: kserve-ci-e2e-test resourceVersion: "15288" uid: 0c1ea2ad-39f7-425d-92c0-b44584cd5cd9 kind: Event lastTimestamp: "2026-07-02T13:29:30Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:29:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:29:30Z" name: success-200-isvc-acf44-predictor-76c47c8649-p5q4f.18be7c1ddff54e01 namespace: kserve-ci-e2e-test resourceVersion: "16547" uid: 99dfc403-d7d1-4e24-ac39-7fb130dc26ac reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:29:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-acf44-predictor-76c47c8649-p5q4f namespace: kserve-ci-e2e-test resourceVersion: "15288" uid: 0c1ea2ad-39f7-425d-92c0-b44584cd5cd9 kind: Event lastTimestamp: "2026-07-02T13:29:33Z" message: 'Readiness probe failed: Get "https://10.132.0.31:8643/healthz": dial tcp 10.132.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:29:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:29:33Z" name: success-200-isvc-acf44-predictor-76c47c8649-p5q4f.18be7c1e95c99800 namespace: kserve-ci-e2e-test resourceVersion: "16662" uid: ebdf3a3e-eabb-41e1-89ab-40f8c9ba3f4c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:27:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-acf44-predictor-76c47c8649 namespace: kserve-ci-e2e-test resourceVersion: "15280" uid: 0ba18d73-5903-4af2-ad8a-2815bd4ba42f kind: Event lastTimestamp: "2026-07-02T13:27:39Z" message: 'Created pod: success-200-isvc-acf44-predictor-76c47c8649-p5q4f' metadata: creationTimestamp: "2026-07-02T13:27:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:27:39Z" name: success-200-isvc-acf44-predictor-76c47c8649.18be7c042628410b namespace: kserve-ci-e2e-test resourceVersion: "15287" uid: 1902414a-57bf-4f7a-a4ae-6f05f8132a52 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:27:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-acf44-predictor namespace: kserve-ci-e2e-test resourceVersion: "15279" uid: e226d765-0575-477f-adbe-294bf115b25f kind: Event lastTimestamp: "2026-07-02T13:27:39Z" message: Scaled up replica set success-200-isvc-acf44-predictor-76c47c8649 from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:27:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:27:39Z" name: success-200-isvc-acf44-predictor.18be7c0422b4e886 namespace: kserve-ci-e2e-test resourceVersion: "15281" uid: 46d8c1d6-41b6-49d1-9263-3a357896ca5d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:27:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-acf44-predictor namespace: kserve-ci-e2e-test resourceVersion: "15298" uid: 0f4829cb-7a63-48b2-87be-abab240de2b2 kind: Event lastTimestamp: "2026-07-02T13:28:39Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:27:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:28:39Z" name: success-200-isvc-acf44-predictor.18be7c07a7d5d6fc namespace: kserve-ci-e2e-test resourceVersion: "16006" uid: 7c59e83e-69b1-471a-be53-897eb45b81cc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:27:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-acf44-predictor namespace: kserve-ci-e2e-test resourceVersion: "15298" uid: 0f4829cb-7a63-48b2-87be-abab240de2b2 kind: Event lastTimestamp: "2026-07-02T13:28:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:27:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:28:39Z" name: success-200-isvc-acf44-predictor.18be7c07a8308852 namespace: kserve-ci-e2e-test resourceVersion: "16008" uid: ae1b4684-db37-4631-9292-4d8f036f9389 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T13:28:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-acf44-predictor namespace: kserve-ci-e2e-test resourceVersion: "15590" uid: 0f4829cb-7a63-48b2-87be-abab240de2b2 kind: Event lastTimestamp: "2026-07-02T13:29:24Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:28:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:29:24Z" name: success-200-isvc-acf44-predictor.18be7c0ea5a3b402 namespace: kserve-ci-e2e-test resourceVersion: "16433" uid: 75f6017c-600a-4f9c-9587-3707d15fd826 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T13:28:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-acf44-predictor namespace: kserve-ci-e2e-test resourceVersion: "15590" uid: 0f4829cb-7a63-48b2-87be-abab240de2b2 kind: Event lastTimestamp: "2026-07-02T13:29:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:28:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:29:24Z" name: success-200-isvc-acf44-predictor.18be7c0ea60238f9 namespace: kserve-ci-e2e-test resourceVersion: "16434" uid: 0ad8a283-8ff1-4fe1-859d-241e1ef7fd26 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T13:27:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-acf44 namespace: kserve-ci-e2e-test resourceVersion: "15267" uid: e073d1de-2ac3-4de0-aaf6-cfd359a15bbe kind: Event lastTimestamp: "2026-07-02T13:29:03Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T13:27:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:29:03Z" name: success-200-isvc-acf44.18be7c0428c9e8aa namespace: kserve-ci-e2e-test resourceVersion: "16218" uid: 3f7fe55e-7af2-4634-8b8f-761739eb98b1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:27:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-acf44 namespace: kserve-ci-e2e-test resourceVersion: "15267" uid: e073d1de-2ac3-4de0-aaf6-cfd359a15bbe kind: Event lastTimestamp: "2026-07-02T13:27:39Z" message: 'Failed to update status for InferenceService "success-200-isvc-acf44": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-acf44": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:27:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:27:39Z" name: success-200-isvc-acf44.18be7c042904c6f2 namespace: kserve-ci-e2e-test resourceVersion: "15300" uid: a5f6e77e-993f-4fd9-a2ff-e48bdd4202b0 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:27:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-acf44 namespace: kserve-ci-e2e-test resourceVersion: "15267" uid: e073d1de-2ac3-4de0-aaf6-cfd359a15bbe kind: Event lastTimestamp: "2026-07-02T13:27:39Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-acf44": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:27:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:27:39Z" name: success-200-isvc-acf44.18be7c0429051dcc namespace: kserve-ci-e2e-test resourceVersion: "15301" uid: 5a1fa890-1eac-4247-8901-df7ee1e70e4e reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:29:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-acf44 namespace: kserve-ci-e2e-test resourceVersion: "16216" uid: e073d1de-2ac3-4de0-aaf6-cfd359a15bbe kind: Event lastTimestamp: "2026-07-02T13:29:03Z" message: InferenceService [success-200-isvc-acf44] is Ready metadata: creationTimestamp: "2026-07-02T13:29:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:29:03Z" name: success-200-isvc-acf44.18be7c179dba62c4 namespace: kserve-ci-e2e-test resourceVersion: "16217" uid: 3345e346-965f-420e-bd0d-37fdd63a2d4f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:39:55.441758Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e2bd2-predictor-6bcb66fb47-n57qc namespace: kserve-ci-e2e-test resourceVersion: "21817" uid: ff4c6996-9d23-4dd8-b001-ed01cf58a717 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-e2bd2-predictor-6bcb66fb47-n57qc to ip-10-0-131-238.ec2.internal metadata: creationTimestamp: "2026-07-02T13:39:55Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:39:55Z" name: success-200-isvc-e2bd2-predictor-6bcb66fb47-n57qc.18be7caf70f8b651 namespace: kserve-ci-e2e-test resourceVersion: "21825" uid: 99627360-65d2-4038-adfc-77a651a6bceb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7f4bb8bc8-mnq2w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:55Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e2bd2-predictor-6bcb66fb47-n57qc namespace: kserve-ci-e2e-test resourceVersion: "21820" uid: ff4c6996-9d23-4dd8-b001-ed01cf58a717 kind: Event lastTimestamp: "2026-07-02T13:39:55Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-e2bd2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T13:39:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:39:55Z" name: success-200-isvc-e2bd2-predictor-6bcb66fb47-n57qc.18be7caf7d3686d4 namespace: kserve-ci-e2e-test resourceVersion: "21866" uid: 15bec68d-490b-4886-8925-a09461da1ef9 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:56Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e2bd2-predictor-6bcb66fb47-n57qc namespace: kserve-ci-e2e-test resourceVersion: "21831" uid: ff4c6996-9d23-4dd8-b001-ed01cf58a717 kind: Event lastTimestamp: "2026-07-02T13:39:56Z" message: Add eth0 [10.132.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:39:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:39:56Z" name: success-200-isvc-e2bd2-predictor-6bcb66fb47-n57qc.18be7cafaea7fc0f namespace: kserve-ci-e2e-test resourceVersion: "21880" uid: f0d146ba-11f3-4096-895e-1cfa3afd6e5f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e2bd2-predictor-6bcb66fb47-n57qc namespace: kserve-ci-e2e-test resourceVersion: "21820" uid: ff4c6996-9d23-4dd8-b001-ed01cf58a717 kind: Event lastTimestamp: "2026-07-02T13:39:56Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1683" already present on machine metadata: creationTimestamp: "2026-07-02T13:39:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:39:56Z" name: success-200-isvc-e2bd2-predictor-6bcb66fb47-n57qc.18be7cafafd08477 namespace: kserve-ci-e2e-test resourceVersion: "21882" uid: 3b240f65-35fa-4f91-a3d3-a104ab7d718d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e2bd2-predictor-6bcb66fb47-n57qc namespace: kserve-ci-e2e-test resourceVersion: "21820" uid: ff4c6996-9d23-4dd8-b001-ed01cf58a717 kind: Event lastTimestamp: "2026-07-02T13:39:56Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T13:39:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:39:56Z" name: success-200-isvc-e2bd2-predictor-6bcb66fb47-n57qc.18be7cafb30c2e86 namespace: kserve-ci-e2e-test resourceVersion: "21884" uid: 5c01f774-ab2e-4cad-9660-a161f001eb9a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e2bd2-predictor-6bcb66fb47-n57qc namespace: kserve-ci-e2e-test resourceVersion: "21820" uid: ff4c6996-9d23-4dd8-b001-ed01cf58a717 kind: Event lastTimestamp: "2026-07-02T13:39:56Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T13:39:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:39:56Z" name: success-200-isvc-e2bd2-predictor-6bcb66fb47-n57qc.18be7cafb3a26cdd namespace: kserve-ci-e2e-test resourceVersion: "21885" uid: 879c5764-8b35-4df5-9ad5-b83f92107bd5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e2bd2-predictor-6bcb66fb47-n57qc namespace: kserve-ci-e2e-test resourceVersion: "21820" uid: ff4c6996-9d23-4dd8-b001-ed01cf58a717 kind: Event lastTimestamp: "2026-07-02T13:39:56Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T13:39:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:39:56Z" name: success-200-isvc-e2bd2-predictor-6bcb66fb47-n57qc.18be7cafb3add756 namespace: kserve-ci-e2e-test resourceVersion: "21886" uid: 49fe5598-e54b-462e-8a32-07a847adab49 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e2bd2-predictor-6bcb66fb47-n57qc namespace: kserve-ci-e2e-test resourceVersion: "21820" uid: ff4c6996-9d23-4dd8-b001-ed01cf58a717 kind: Event lastTimestamp: "2026-07-02T13:39:56Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T13:39:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:39:56Z" name: success-200-isvc-e2bd2-predictor-6bcb66fb47-n57qc.18be7cafb737a2e0 namespace: kserve-ci-e2e-test resourceVersion: "21890" uid: 4989d9d8-353d-4fa3-ae26-97bc4f49f8e1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e2bd2-predictor-6bcb66fb47-n57qc namespace: kserve-ci-e2e-test resourceVersion: "21820" uid: ff4c6996-9d23-4dd8-b001-ed01cf58a717 kind: Event lastTimestamp: "2026-07-02T13:39:56Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:39:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:39:56Z" name: success-200-isvc-e2bd2-predictor-6bcb66fb47-n57qc.18be7cafb7b39ddd namespace: kserve-ci-e2e-test resourceVersion: "21891" uid: dc45ebcf-cb08-49d0-8d55-2b072ff04c13 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:39:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e2bd2-predictor-6bcb66fb47-n57qc namespace: kserve-ci-e2e-test resourceVersion: "21820" uid: ff4c6996-9d23-4dd8-b001-ed01cf58a717 kind: Event lastTimestamp: "2026-07-02T13:40:33Z" message: 'Readiness probe failed: dial tcp 10.132.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:39:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:40:33Z" name: success-200-isvc-e2bd2-predictor-6bcb66fb47-n57qc.18be7caff932d2e9 namespace: kserve-ci-e2e-test resourceVersion: "22257" uid: 78e1778f-dbac-48f7-80a0-3d6ac426045e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:49:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e2bd2-predictor-6bcb66fb47-n57qc namespace: kserve-ci-e2e-test resourceVersion: "21820" uid: ff4c6996-9d23-4dd8-b001-ed01cf58a717 kind: Event lastTimestamp: "2026-07-02T13:49:15Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T13:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:49:15Z" name: success-200-isvc-e2bd2-predictor-6bcb66fb47-n57qc.18be7d31d3869a24 namespace: kserve-ci-e2e-test resourceVersion: "26508" uid: b0a54d88-1880-4e60-ad9f-2786f92a0094 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:49:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e2bd2-predictor-6bcb66fb47-n57qc namespace: kserve-ci-e2e-test resourceVersion: "21820" uid: ff4c6996-9d23-4dd8-b001-ed01cf58a717 kind: Event lastTimestamp: "2026-07-02T13:49:15Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:49:15Z" name: success-200-isvc-e2bd2-predictor-6bcb66fb47-n57qc.18be7d31d38bc5df namespace: kserve-ci-e2e-test resourceVersion: "26511" uid: 7e2c04ec-aff9-478c-9fa2-f743e1cbdde9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:55Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-e2bd2-predictor-6bcb66fb47 namespace: kserve-ci-e2e-test resourceVersion: "21800" uid: c82c3367-7bc5-498d-9e62-b6c766bca6ba kind: Event lastTimestamp: "2026-07-02T13:39:55Z" message: 'Created pod: success-200-isvc-e2bd2-predictor-6bcb66fb47-n57qc' metadata: creationTimestamp: "2026-07-02T13:39:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:39:55Z" name: success-200-isvc-e2bd2-predictor-6bcb66fb47.18be7caf7089aec6 namespace: kserve-ci-e2e-test resourceVersion: "21819" uid: 0eaa2e7b-8e1a-462c-8a06-6c139fca6256 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:55Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-e2bd2-predictor namespace: kserve-ci-e2e-test resourceVersion: "21795" uid: 024f7019-a2c5-4b3c-ae75-866f4e910538 kind: Event lastTimestamp: "2026-07-02T13:39:55Z" message: Scaled up replica set success-200-isvc-e2bd2-predictor-6bcb66fb47 from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:39:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:39:55Z" name: success-200-isvc-e2bd2-predictor.18be7caf6e988c53 namespace: kserve-ci-e2e-test resourceVersion: "21805" uid: 09f79032-0dab-4f50-8026-9e447354330e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:40:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e2bd2-predictor namespace: kserve-ci-e2e-test resourceVersion: "21799" uid: 5c6160a4-df2c-48ea-af93-520f679dd95f kind: Event lastTimestamp: "2026-07-02T13:40:25Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:40:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:40:25Z" name: success-200-isvc-e2bd2-predictor.18be7cb2ed1fae67 namespace: kserve-ci-e2e-test resourceVersion: "22189" uid: 4d4b52a2-919c-42f7-a609-9c44ea7f8e2a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:40:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e2bd2-predictor namespace: kserve-ci-e2e-test resourceVersion: "21799" uid: 5c6160a4-df2c-48ea-af93-520f679dd95f kind: Event lastTimestamp: "2026-07-02T13:40:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:40:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:40:25Z" name: success-200-isvc-e2bd2-predictor.18be7cb2ed7a1413 namespace: kserve-ci-e2e-test resourceVersion: "22190" uid: 752939c1-3946-4b88-99f3-63555062f699 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:40:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e2bd2-predictor namespace: kserve-ci-e2e-test resourceVersion: "22022" uid: 5c6160a4-df2c-48ea-af93-520f679dd95f kind: Event lastTimestamp: "2026-07-02T13:41:10Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:40:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:41:10Z" name: success-200-isvc-e2bd2-predictor.18be7cb9eaa7a6c6 namespace: kserve-ci-e2e-test resourceVersion: "22760" uid: 0ab247da-8f3d-41de-88c3-cbf5ba5b997a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:40:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e2bd2-predictor namespace: kserve-ci-e2e-test resourceVersion: "22022" uid: 5c6160a4-df2c-48ea-af93-520f679dd95f kind: Event lastTimestamp: "2026-07-02T13:41:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:40:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:41:10Z" name: success-200-isvc-e2bd2-predictor.18be7cb9eb0408e5 namespace: kserve-ci-e2e-test resourceVersion: "22761" uid: 996d337a-60aa-4aae-94f1-a6939d4a754f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:39:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e2bd2 namespace: kserve-ci-e2e-test resourceVersion: "21776" uid: 366b67a0-dd46-4363-b6e5-e4ee0aff6b0c kind: Event lastTimestamp: "2026-07-02T13:40:43Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T13:39:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:40:43Z" name: success-200-isvc-e2bd2.18be7caf6f16cdad namespace: kserve-ci-e2e-test resourceVersion: "22506" uid: aecd5065-c8a2-47f4-bbff-f2eee28bff50 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:40:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e2bd2 namespace: kserve-ci-e2e-test resourceVersion: "22504" uid: 366b67a0-dd46-4363-b6e5-e4ee0aff6b0c kind: Event lastTimestamp: "2026-07-02T13:40:43Z" message: InferenceService [success-200-isvc-e2bd2] is Ready metadata: creationTimestamp: "2026-07-02T13:40:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:40:43Z" name: success-200-isvc-e2bd2.18be7cbab39fa37c namespace: kserve-ci-e2e-test resourceVersion: "22505" uid: 269c8668-bcd3-487b-9527-26ec2c16f318 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:30:10.077847Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f369f-predictor-7957879bc7-pvbxz namespace: kserve-ci-e2e-test resourceVersion: "17057" uid: beb4e33f-3c05-4075-b592-f5951759c36a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-f369f-predictor-7957879bc7-pvbxz to ip-10-0-131-238.ec2.internal metadata: creationTimestamp: "2026-07-02T13:30:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:30:10Z" name: success-200-isvc-f369f-predictor-7957879bc7-pvbxz.18be7c2726904a67 namespace: kserve-ci-e2e-test resourceVersion: "17064" uid: f9eae016-91c3-4f2b-92d5-89ad255b6723 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7f4bb8bc8-mnq2w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:30:10Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f369f-predictor-7957879bc7-pvbxz namespace: kserve-ci-e2e-test resourceVersion: "17062" uid: beb4e33f-3c05-4075-b592-f5951759c36a kind: Event lastTimestamp: "2026-07-02T13:30:10Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-f369f-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T13:30:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:30:10Z" name: success-200-isvc-f369f-predictor-7957879bc7-pvbxz.18be7c2730243697 namespace: kserve-ci-e2e-test resourceVersion: "17116" uid: 0d66dc33-775a-4da1-b1df-3fe5bf71dd4a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:30:11Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f369f-predictor-7957879bc7-pvbxz namespace: kserve-ci-e2e-test resourceVersion: "17079" uid: beb4e33f-3c05-4075-b592-f5951759c36a kind: Event lastTimestamp: "2026-07-02T13:30:11Z" message: Add eth0 [10.132.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:30:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:30:11Z" name: success-200-isvc-f369f-predictor-7957879bc7-pvbxz.18be7c2764b0ad82 namespace: kserve-ci-e2e-test resourceVersion: "17144" uid: 4bfa4f0d-c501-4a95-92d5-f40448bf7142 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:30:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f369f-predictor-7957879bc7-pvbxz namespace: kserve-ci-e2e-test resourceVersion: "17062" uid: beb4e33f-3c05-4075-b592-f5951759c36a kind: Event lastTimestamp: "2026-07-02T13:30:11Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1683" already present on machine metadata: creationTimestamp: "2026-07-02T13:30:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:30:11Z" name: success-200-isvc-f369f-predictor-7957879bc7-pvbxz.18be7c276626ca6e namespace: kserve-ci-e2e-test resourceVersion: "17147" uid: e5128bc5-1f73-4a84-b34d-68fa479f51bd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:30:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f369f-predictor-7957879bc7-pvbxz namespace: kserve-ci-e2e-test resourceVersion: "17062" uid: beb4e33f-3c05-4075-b592-f5951759c36a kind: Event lastTimestamp: "2026-07-02T13:30:11Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T13:30:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:30:11Z" name: success-200-isvc-f369f-predictor-7957879bc7-pvbxz.18be7c27696c0ef7 namespace: kserve-ci-e2e-test resourceVersion: "17148" uid: a8b09f61-8dfe-4370-8ddf-eea223fce721 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:30:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f369f-predictor-7957879bc7-pvbxz namespace: kserve-ci-e2e-test resourceVersion: "17062" uid: beb4e33f-3c05-4075-b592-f5951759c36a kind: Event lastTimestamp: "2026-07-02T13:30:11Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T13:30:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:30:11Z" name: success-200-isvc-f369f-predictor-7957879bc7-pvbxz.18be7c2769dfee96 namespace: kserve-ci-e2e-test resourceVersion: "17149" uid: 4bb8523d-b483-4b0b-9d19-43670d61f134 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:30:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f369f-predictor-7957879bc7-pvbxz namespace: kserve-ci-e2e-test resourceVersion: "17062" uid: beb4e33f-3c05-4075-b592-f5951759c36a kind: Event lastTimestamp: "2026-07-02T13:30:11Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T13:30:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:30:11Z" name: success-200-isvc-f369f-predictor-7957879bc7-pvbxz.18be7c2769eb4d58 namespace: kserve-ci-e2e-test resourceVersion: "17150" uid: 68c3ba1b-5a84-4714-b940-e87dbc7da16c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:30:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f369f-predictor-7957879bc7-pvbxz namespace: kserve-ci-e2e-test resourceVersion: "17062" uid: beb4e33f-3c05-4075-b592-f5951759c36a kind: Event lastTimestamp: "2026-07-02T13:30:11Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T13:30:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:30:11Z" name: success-200-isvc-f369f-predictor-7957879bc7-pvbxz.18be7c276d15ed8a namespace: kserve-ci-e2e-test resourceVersion: "17151" uid: 73bd82e5-6315-416c-ab10-884afcd2c15b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:30:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f369f-predictor-7957879bc7-pvbxz namespace: kserve-ci-e2e-test resourceVersion: "17062" uid: beb4e33f-3c05-4075-b592-f5951759c36a kind: Event lastTimestamp: "2026-07-02T13:30:11Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:30:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:30:11Z" name: success-200-isvc-f369f-predictor-7957879bc7-pvbxz.18be7c276d8c2637 namespace: kserve-ci-e2e-test resourceVersion: "17152" uid: 2581f712-1793-4a5a-9541-18e515147c6a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:30:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f369f-predictor-7957879bc7-pvbxz namespace: kserve-ci-e2e-test resourceVersion: "17062" uid: beb4e33f-3c05-4075-b592-f5951759c36a kind: Event lastTimestamp: "2026-07-02T13:30:48Z" message: 'Readiness probe failed: dial tcp 10.132.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:30:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:30:48Z" name: success-200-isvc-f369f-predictor-7957879bc7-pvbxz.18be7c27d0d636a4 namespace: kserve-ci-e2e-test resourceVersion: "17535" uid: 1766ef85-97fb-4b0d-9ad4-935d6d86fde6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f369f-predictor-7957879bc7-pvbxz namespace: kserve-ci-e2e-test resourceVersion: "17062" uid: beb4e33f-3c05-4075-b592-f5951759c36a kind: Event lastTimestamp: "2026-07-02T13:39:24Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T13:39:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:39:24Z" name: success-200-isvc-f369f-predictor-7957879bc7-pvbxz.18be7ca85497d6ed namespace: kserve-ci-e2e-test resourceVersion: "21373" uid: 1e083f4e-968b-4979-8a9d-6892ee8122d8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:39:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f369f-predictor-7957879bc7-pvbxz namespace: kserve-ci-e2e-test resourceVersion: "17062" uid: beb4e33f-3c05-4075-b592-f5951759c36a kind: Event lastTimestamp: "2026-07-02T13:39:24Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:39:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:39:24Z" name: success-200-isvc-f369f-predictor-7957879bc7-pvbxz.18be7ca85498a1ed namespace: kserve-ci-e2e-test resourceVersion: "21376" uid: a8f2d807-0090-4f37-9731-4e5dd63dda99 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-238.ec2.internal source: component: kubelet host: ip-10-0-131-238.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:30:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-f369f-predictor-7957879bc7 namespace: kserve-ci-e2e-test resourceVersion: "17048" uid: 9b390136-dc48-4de2-8e77-1dc69c69e4a3 kind: Event lastTimestamp: "2026-07-02T13:30:10Z" message: 'Created pod: success-200-isvc-f369f-predictor-7957879bc7-pvbxz' metadata: creationTimestamp: "2026-07-02T13:30:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:30:10Z" name: success-200-isvc-f369f-predictor-7957879bc7.18be7c27261b811e namespace: kserve-ci-e2e-test resourceVersion: "17061" uid: 8ea30da6-5142-427c-8d6f-e441b11348e9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:30:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-f369f-predictor namespace: kserve-ci-e2e-test resourceVersion: "17040" uid: 23503ef1-a43a-4cee-b42f-735e64629f07 kind: Event lastTimestamp: "2026-07-02T13:30:10Z" message: Scaled up replica set success-200-isvc-f369f-predictor-7957879bc7 from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:30:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:30:10Z" name: success-200-isvc-f369f-predictor.18be7c2724931718 namespace: kserve-ci-e2e-test resourceVersion: "17049" uid: 1da51f49-2c6a-4654-aec1-6c19da78d82e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:30:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f369f-predictor namespace: kserve-ci-e2e-test resourceVersion: "17043" uid: ac01dc24-1254-498d-8b71-77e3ef041bf6 kind: Event lastTimestamp: "2026-07-02T13:30:40Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:30:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:30:40Z" name: success-200-isvc-f369f-predictor.18be7c2aa32e836c namespace: kserve-ci-e2e-test resourceVersion: "17456" uid: 1ae9ce2e-3dd5-4227-9c94-b3ba8f20c7bc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:30:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f369f-predictor namespace: kserve-ci-e2e-test resourceVersion: "17043" uid: ac01dc24-1254-498d-8b71-77e3ef041bf6 kind: Event lastTimestamp: "2026-07-02T13:30:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:30:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:30:40Z" name: success-200-isvc-f369f-predictor.18be7c2aa3867c38 namespace: kserve-ci-e2e-test resourceVersion: "17457" uid: f668b22b-b451-4022-833e-d23c6de2497a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:30:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f369f-predictor namespace: kserve-ci-e2e-test resourceVersion: "17301" uid: ac01dc24-1254-498d-8b71-77e3ef041bf6 kind: Event lastTimestamp: "2026-07-02T13:31:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:30:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:31:25Z" name: success-200-isvc-f369f-predictor.18be7c31a0fa2a87 namespace: kserve-ci-e2e-test resourceVersion: "17853" uid: bf5afed4-aa8f-4d52-8a24-cfb22f908613 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:30:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f369f-predictor namespace: kserve-ci-e2e-test resourceVersion: "17301" uid: ac01dc24-1254-498d-8b71-77e3ef041bf6 kind: Event lastTimestamp: "2026-07-02T13:31:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:30:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:31:25Z" name: success-200-isvc-f369f-predictor.18be7c31a15a450d namespace: kserve-ci-e2e-test resourceVersion: "17854" uid: 92efc913-877e-44eb-bc4f-840a61bce73a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:30:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f369f namespace: kserve-ci-e2e-test resourceVersion: "17018" uid: 8709a67a-20a1-422c-8840-164218a9813b kind: Event lastTimestamp: "2026-07-02T13:30:59Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T13:30:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:30:59Z" name: success-200-isvc-f369f.18be7c272492c92a namespace: kserve-ci-e2e-test resourceVersion: "17621" uid: 19802b70-d521-4c69-9db4-3679b440ede2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:30:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f369f namespace: kserve-ci-e2e-test resourceVersion: "17618" uid: 8709a67a-20a1-422c-8840-164218a9813b kind: Event lastTimestamp: "2026-07-02T13:30:59Z" message: InferenceService [success-200-isvc-f369f] is Ready metadata: creationTimestamp: "2026-07-02T13:30:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:30:59Z" name: success-200-isvc-f369f.18be7c328b5bd145 namespace: kserve-ci-e2e-test resourceVersion: "17619" uid: 2dd94135-f551-4b08-b6a7-d389ba3ba30a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:30:30.313899Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-383ed-79cc5cfd9b-cjs9c namespace: kserve-ci-e2e-test resourceVersion: "17353" uid: d54e0e73-1c73-430a-a65a-aa0c530d90d4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-383ed-79cc5cfd9b-cjs9c to ip-10-0-133-122.ec2.internal metadata: creationTimestamp: "2026-07-02T13:30:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:30:30Z" name: switch-graph-383ed-79cc5cfd9b-cjs9c.18be7c2bdcb9f903 namespace: kserve-ci-e2e-test resourceVersion: "17358" uid: 06f887d1-eeff-4898-b460-6b2eb1c8b6af reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7f4bb8bc8-mnq2w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:30:30Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-383ed-79cc5cfd9b-cjs9c namespace: kserve-ci-e2e-test resourceVersion: "17354" uid: d54e0e73-1c73-430a-a65a-aa0c530d90d4 kind: Event lastTimestamp: "2026-07-02T13:30:30Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-383ed-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T13:30:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:30:30Z" name: switch-graph-383ed-79cc5cfd9b-cjs9c.18be7c2be7689b64 namespace: kserve-ci-e2e-test resourceVersion: "17371" uid: 80261599-b090-405e-9f5f-19c8bf5e7434 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:30:31Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-383ed-79cc5cfd9b-cjs9c namespace: kserve-ci-e2e-test resourceVersion: "17362" uid: d54e0e73-1c73-430a-a65a-aa0c530d90d4 kind: Event lastTimestamp: "2026-07-02T13:30:31Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:30:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:30:31Z" name: switch-graph-383ed-79cc5cfd9b-cjs9c.18be7c2c19f4b61a namespace: kserve-ci-e2e-test resourceVersion: "17378" uid: 85569b33-e043-408d-b149-9bb5ee7ebaf1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:30:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-383ed} kind: Pod name: switch-graph-383ed-79cc5cfd9b-cjs9c namespace: kserve-ci-e2e-test resourceVersion: "17354" uid: d54e0e73-1c73-430a-a65a-aa0c530d90d4 kind: Event lastTimestamp: "2026-07-02T13:30:31Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:9f703da9f2f5afd108b6816a57feadccff9213d4d8437a63e732fa3fbaa3fa9d" already present on machine metadata: creationTimestamp: "2026-07-02T13:30:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:30:31Z" name: switch-graph-383ed-79cc5cfd9b-cjs9c.18be7c2c1b3b602e namespace: kserve-ci-e2e-test resourceVersion: "17380" uid: f42058c4-0041-4489-ac7b-259da4429cbc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:30:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-383ed} kind: Pod name: switch-graph-383ed-79cc5cfd9b-cjs9c namespace: kserve-ci-e2e-test resourceVersion: "17354" uid: d54e0e73-1c73-430a-a65a-aa0c530d90d4 kind: Event lastTimestamp: "2026-07-02T13:30:31Z" message: 'Created container: switch-graph-383ed' metadata: creationTimestamp: "2026-07-02T13:30:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:30:31Z" name: switch-graph-383ed-79cc5cfd9b-cjs9c.18be7c2c1e942976 namespace: kserve-ci-e2e-test resourceVersion: "17381" uid: 8f803fe0-4b20-4694-9db1-5a18734d876f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:30:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-383ed} kind: Pod name: switch-graph-383ed-79cc5cfd9b-cjs9c namespace: kserve-ci-e2e-test resourceVersion: "17354" uid: d54e0e73-1c73-430a-a65a-aa0c530d90d4 kind: Event lastTimestamp: "2026-07-02T13:30:31Z" message: Started container switch-graph-383ed metadata: creationTimestamp: "2026-07-02T13:30:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:30:31Z" name: switch-graph-383ed-79cc5cfd9b-cjs9c.18be7c2c1f041eca namespace: kserve-ci-e2e-test resourceVersion: "17382" uid: f972e49a-e2b9-42b2-b2a2-bf2993ca2076 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:38:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-383ed} kind: Pod name: switch-graph-383ed-79cc5cfd9b-cjs9c namespace: kserve-ci-e2e-test resourceVersion: "17354" uid: d54e0e73-1c73-430a-a65a-aa0c530d90d4 kind: Event lastTimestamp: "2026-07-02T13:38:45Z" message: Stopping container switch-graph-383ed metadata: creationTimestamp: "2026-07-02T13:38:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:38:45Z" name: switch-graph-383ed-79cc5cfd9b-cjs9c.18be7c9f0ad65339 namespace: kserve-ci-e2e-test resourceVersion: "20864" uid: 1dc01758-06b9-4ddf-93bb-bc92f9148c46 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:38:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-383ed} kind: Pod name: switch-graph-383ed-79cc5cfd9b-cjs9c namespace: kserve-ci-e2e-test resourceVersion: "17354" uid: d54e0e73-1c73-430a-a65a-aa0c530d90d4 kind: Event lastTimestamp: "2026-07-02T13:39:12Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T13:38:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:39:12Z" name: switch-graph-383ed-79cc5cfd9b-cjs9c.18be7c9fb59fa766 namespace: kserve-ci-e2e-test resourceVersion: "21206" uid: 7b390d52-9e96-444b-afb3-58834beb3e44 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:30:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-383ed-79cc5cfd9b namespace: kserve-ci-e2e-test resourceVersion: "17350" uid: 13a1538e-9850-4a95-b040-cbeb46fdc1ff kind: Event lastTimestamp: "2026-07-02T13:30:30Z" message: 'Created pod: switch-graph-383ed-79cc5cfd9b-cjs9c' metadata: creationTimestamp: "2026-07-02T13:30:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:30:30Z" name: switch-graph-383ed-79cc5cfd9b.18be7c2bdc5cd87f namespace: kserve-ci-e2e-test resourceVersion: "17355" uid: 75dd704a-0457-4083-8410-cb04d4c79fed reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:30:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-383ed namespace: kserve-ci-e2e-test resourceVersion: "17348" uid: 7c733879-861e-4d5e-9dc6-17577eaf6364 kind: Event lastTimestamp: "2026-07-02T13:30:30Z" message: Scaled up replica set switch-graph-383ed-79cc5cfd9b from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:30:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:30:30Z" name: switch-graph-383ed.18be7c2bdb77f3a6 namespace: kserve-ci-e2e-test resourceVersion: "17351" uid: 8eaef2a5-42aa-405a-ad5f-3a329a3e2753 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:30:30Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-383ed namespace: kserve-ci-e2e-test resourceVersion: "17367" uid: 32654163-66bf-4acb-b932-d787f4dc5836 kind: Event lastTimestamp: "2026-07-02T13:30:30Z" message: InferenceGraph [switch-graph-383ed] is Ready metadata: creationTimestamp: "2026-07-02T13:30:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:30:30Z" name: switch-graph-383ed.18be7c2be00aafc6 namespace: kserve-ci-e2e-test resourceVersion: "17369" uid: 7822a360-e5c6-4f87-966f-9e0049dc43ab reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:30:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-383ed namespace: kserve-ci-e2e-test resourceVersion: "17349" uid: 14df4522-8cbc-4a12-ac3a-b1ca11609694 kind: Event lastTimestamp: "2026-07-02T13:31:00Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:30:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:31:00Z" name: switch-graph-383ed.18be7c2f59c6ffa5 namespace: kserve-ci-e2e-test resourceVersion: "17635" uid: be039e18-6437-4de5-ab16-8c760afd3018 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:30:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-383ed namespace: kserve-ci-e2e-test resourceVersion: "17349" uid: 14df4522-8cbc-4a12-ac3a-b1ca11609694 kind: Event lastTimestamp: "2026-07-02T13:31:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:30:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:31:00Z" name: switch-graph-383ed.18be7c2f5a223139 namespace: kserve-ci-e2e-test resourceVersion: "17636" uid: e48eb980-cfc3-43c4-83d0-782b82901a76 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:50:50.106136Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-7f184-7676f88dd9-z4frx namespace: kserve-ci-e2e-test resourceVersion: "27708" uid: 175b54bb-8dce-4f40-9044-70aef274b1ae kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-7f184-7676f88dd9-z4frx to ip-10-0-133-122.ec2.internal metadata: creationTimestamp: "2026-07-02T13:50:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:50:50Z" name: switch-graph-7f184-7676f88dd9-z4frx.18be7d47de025791 namespace: kserve-ci-e2e-test resourceVersion: "27713" uid: 0a652661-68f2-441e-8a6a-d2163c33be17 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7f4bb8bc8-mnq2w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:50:50Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-7f184-7676f88dd9-z4frx namespace: kserve-ci-e2e-test resourceVersion: "27710" uid: 175b54bb-8dce-4f40-9044-70aef274b1ae kind: Event lastTimestamp: "2026-07-02T13:50:50Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-7f184-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T13:50:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:50:50Z" name: switch-graph-7f184-7676f88dd9-z4frx.18be7d47e8a80c68 namespace: kserve-ci-e2e-test resourceVersion: "27729" uid: 313f7eaf-e1f1-4063-aa93-003481f224e5 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:50:51Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-7f184-7676f88dd9-z4frx namespace: kserve-ci-e2e-test resourceVersion: "27720" uid: 175b54bb-8dce-4f40-9044-70aef274b1ae kind: Event lastTimestamp: "2026-07-02T13:50:51Z" message: Add eth0 [10.134.0.48/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:50:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:50:51Z" name: switch-graph-7f184-7676f88dd9-z4frx.18be7d481b57c37d namespace: kserve-ci-e2e-test resourceVersion: "27736" uid: e5861ba2-3394-46a0-8efc-66ee7ee43e58 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:50:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-7f184} kind: Pod name: switch-graph-7f184-7676f88dd9-z4frx namespace: kserve-ci-e2e-test resourceVersion: "27710" uid: 175b54bb-8dce-4f40-9044-70aef274b1ae kind: Event lastTimestamp: "2026-07-02T13:50:51Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:9f703da9f2f5afd108b6816a57feadccff9213d4d8437a63e732fa3fbaa3fa9d" already present on machine metadata: creationTimestamp: "2026-07-02T13:50:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:50:51Z" name: switch-graph-7f184-7676f88dd9-z4frx.18be7d481cb7a3b7 namespace: kserve-ci-e2e-test resourceVersion: "27739" uid: 3628cd54-e413-40af-969b-1417c8c6b868 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:50:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-7f184} kind: Pod name: switch-graph-7f184-7676f88dd9-z4frx namespace: kserve-ci-e2e-test resourceVersion: "27710" uid: 175b54bb-8dce-4f40-9044-70aef274b1ae kind: Event lastTimestamp: "2026-07-02T13:50:51Z" message: 'Created container: switch-graph-7f184' metadata: creationTimestamp: "2026-07-02T13:50:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:50:51Z" name: switch-graph-7f184-7676f88dd9-z4frx.18be7d481fed8dfe namespace: kserve-ci-e2e-test resourceVersion: "27740" uid: 58c0daf5-b66f-4d0a-be8f-b57acc2cf4da reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:50:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-7f184} kind: Pod name: switch-graph-7f184-7676f88dd9-z4frx namespace: kserve-ci-e2e-test resourceVersion: "27710" uid: 175b54bb-8dce-4f40-9044-70aef274b1ae kind: Event lastTimestamp: "2026-07-02T13:50:51Z" message: Started container switch-graph-7f184 metadata: creationTimestamp: "2026-07-02T13:50:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:50:51Z" name: switch-graph-7f184-7676f88dd9-z4frx.18be7d48206b8e9f namespace: kserve-ci-e2e-test resourceVersion: "27741" uid: 5eb294ae-327b-4efa-a9c4-f1cbd9fd0cb9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:07:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-7f184} kind: Pod name: switch-graph-7f184-7676f88dd9-z4frx namespace: kserve-ci-e2e-test resourceVersion: "27710" uid: 175b54bb-8dce-4f40-9044-70aef274b1ae kind: Event lastTimestamp: "2026-07-02T14:07:09Z" message: Stopping container switch-graph-7f184 metadata: creationTimestamp: "2026-07-02T14:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:07:09Z" name: switch-graph-7f184-7676f88dd9-z4frx.18be7e2be33c08fc namespace: kserve-ci-e2e-test resourceVersion: "34629" uid: 525051de-f599-473e-99f7-1e406ca6aef3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T14:07:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-7f184} kind: Pod name: switch-graph-7f184-7676f88dd9-z4frx namespace: kserve-ci-e2e-test resourceVersion: "27710" uid: 175b54bb-8dce-4f40-9044-70aef274b1ae kind: Event lastTimestamp: "2026-07-02T14:07:33Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T14:07:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:07:33Z" name: switch-graph-7f184-7676f88dd9-z4frx.18be7e2cc390b42e namespace: kserve-ci-e2e-test resourceVersion: "34874" uid: f6dc8e59-2034-4a98-9c5e-b282b8b2e112 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:50:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-7f184-7676f88dd9 namespace: kserve-ci-e2e-test resourceVersion: "27705" uid: 4b47033c-3d94-466e-a236-b0a392b29d9e kind: Event lastTimestamp: "2026-07-02T13:50:50Z" message: 'Created pod: switch-graph-7f184-7676f88dd9-z4frx' metadata: creationTimestamp: "2026-07-02T13:50:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:50:50Z" name: switch-graph-7f184-7676f88dd9.18be7d47dd94caae namespace: kserve-ci-e2e-test resourceVersion: "27709" uid: 8f6160e8-bb7f-49c1-82ab-1977996db06c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:50:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-7f184 namespace: kserve-ci-e2e-test resourceVersion: "27703" uid: 3ff03ea3-c279-48c9-81d8-74cc0d7f605f kind: Event lastTimestamp: "2026-07-02T13:50:50Z" message: Scaled up replica set switch-graph-7f184-7676f88dd9 from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:50:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:50:50Z" name: switch-graph-7f184.18be7d47dce88218 namespace: kserve-ci-e2e-test resourceVersion: "27706" uid: f372cff1-bc5b-4cba-8da7-ea9e11924ea0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:50:50Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-7f184 namespace: kserve-ci-e2e-test resourceVersion: "27722" uid: ba68e3a4-cf19-4930-b5d3-65a65299c804 kind: Event lastTimestamp: "2026-07-02T13:50:50Z" message: InferenceGraph [switch-graph-7f184] is Ready metadata: creationTimestamp: "2026-07-02T13:50:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:50:50Z" name: switch-graph-7f184.18be7d47e15ab9d8 namespace: kserve-ci-e2e-test resourceVersion: "27724" uid: 82e40ac8-364e-4382-8ae7-903968ff2fef reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:51:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-7f184 namespace: kserve-ci-e2e-test resourceVersion: "27704" uid: 3c0b9a0f-be61-4503-a70b-0756f59d2078 kind: Event lastTimestamp: "2026-07-02T13:51:35Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:51:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:51:35Z" name: switch-graph-7f184.18be7d4b5b58a98c namespace: kserve-ci-e2e-test resourceVersion: "28147" uid: 9d8d3966-f00e-4ffa-8679-4430fce3f7c2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:51:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-7f184 namespace: kserve-ci-e2e-test resourceVersion: "27704" uid: 3c0b9a0f-be61-4503-a70b-0756f59d2078 kind: Event lastTimestamp: "2026-07-02T13:51:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:51:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:51:35Z" name: switch-graph-7f184.18be7d4b5bb8a874 namespace: kserve-ci-e2e-test resourceVersion: "28148" uid: 755d775e-6252-449d-bed7-17612a8fed1d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:29:19.852135Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-acf44-5486c59c4d-rtpd8 namespace: kserve-ci-e2e-test resourceVersion: "16353" uid: e68fbe03-b9cf-49d0-8e32-d2d966afc588 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-acf44-5486c59c4d-rtpd8 to ip-10-0-133-122.ec2.internal metadata: creationTimestamp: "2026-07-02T13:29:19Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:29:19Z" name: switch-graph-acf44-5486c59c4d-rtpd8.18be7c1b74e0b5d4 namespace: kserve-ci-e2e-test resourceVersion: "16360" uid: 67a94d6a-1a04-4ef0-ba7e-db5a4a7bc0ef reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7f4bb8bc8-mnq2w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:29:20Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-acf44-5486c59c4d-rtpd8 namespace: kserve-ci-e2e-test resourceVersion: "16357" uid: e68fbe03-b9cf-49d0-8e32-d2d966afc588 kind: Event lastTimestamp: "2026-07-02T13:29:20Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-acf44-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T13:29:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:29:20Z" name: switch-graph-acf44-5486c59c4d-rtpd8.18be7c1b8612b536 namespace: kserve-ci-e2e-test resourceVersion: "16374" uid: e530663e-a344-457a-b065-f6fef0c4af27 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:29:20Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-acf44-5486c59c4d-rtpd8 namespace: kserve-ci-e2e-test resourceVersion: "16363" uid: e68fbe03-b9cf-49d0-8e32-d2d966afc588 kind: Event lastTimestamp: "2026-07-02T13:29:20Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:29:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:29:20Z" name: switch-graph-acf44-5486c59c4d-rtpd8.18be7c1bb1f92041 namespace: kserve-ci-e2e-test resourceVersion: "16382" uid: e2cf534c-f817-448c-b488-b1992149b81e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:29:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-acf44} kind: Pod name: switch-graph-acf44-5486c59c4d-rtpd8 namespace: kserve-ci-e2e-test resourceVersion: "16357" uid: e68fbe03-b9cf-49d0-8e32-d2d966afc588 kind: Event lastTimestamp: "2026-07-02T13:29:20Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:9f703da9f2f5afd108b6816a57feadccff9213d4d8437a63e732fa3fbaa3fa9d" metadata: creationTimestamp: "2026-07-02T13:29:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:29:20Z" name: switch-graph-acf44-5486c59c4d-rtpd8.18be7c1bb32d7e26 namespace: kserve-ci-e2e-test resourceVersion: "16384" uid: 9784e875-ab69-4425-a104-2e185b52e9ef reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:29:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-acf44} kind: Pod name: switch-graph-acf44-5486c59c4d-rtpd8 namespace: kserve-ci-e2e-test resourceVersion: "16357" uid: e68fbe03-b9cf-49d0-8e32-d2d966afc588 kind: Event lastTimestamp: "2026-07-02T13:29:23Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:9f703da9f2f5afd108b6816a57feadccff9213d4d8437a63e732fa3fbaa3fa9d" in 2.401s (2.401s including waiting). Image size: 218070105 bytes.' metadata: creationTimestamp: "2026-07-02T13:29:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:29:23Z" name: switch-graph-acf44-5486c59c4d-rtpd8.18be7c1c424ad022 namespace: kserve-ci-e2e-test resourceVersion: "16402" uid: ede27a65-14da-4617-b94b-30f444d951a7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:29:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-acf44} kind: Pod name: switch-graph-acf44-5486c59c4d-rtpd8 namespace: kserve-ci-e2e-test resourceVersion: "16357" uid: e68fbe03-b9cf-49d0-8e32-d2d966afc588 kind: Event lastTimestamp: "2026-07-02T13:29:23Z" message: 'Created container: switch-graph-acf44' metadata: creationTimestamp: "2026-07-02T13:29:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:29:23Z" name: switch-graph-acf44-5486c59c4d-rtpd8.18be7c1c45cef967 namespace: kserve-ci-e2e-test resourceVersion: "16403" uid: 42c471d3-e6f2-459b-891a-2b9517e46da6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:29:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-acf44} kind: Pod name: switch-graph-acf44-5486c59c4d-rtpd8 namespace: kserve-ci-e2e-test resourceVersion: "16357" uid: e68fbe03-b9cf-49d0-8e32-d2d966afc588 kind: Event lastTimestamp: "2026-07-02T13:29:23Z" message: Started container switch-graph-acf44 metadata: creationTimestamp: "2026-07-02T13:29:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:29:23Z" name: switch-graph-acf44-5486c59c4d-rtpd8.18be7c1c46871cb4 namespace: kserve-ci-e2e-test resourceVersion: "16404" uid: bf418063-28d9-4aa1-90f0-5630caedaf32 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:29:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-acf44} kind: Pod name: switch-graph-acf44-5486c59c4d-rtpd8 namespace: kserve-ci-e2e-test resourceVersion: "16357" uid: e68fbe03-b9cf-49d0-8e32-d2d966afc588 kind: Event lastTimestamp: "2026-07-02T13:29:30Z" message: Stopping container switch-graph-acf44 metadata: creationTimestamp: "2026-07-02T13:29:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:29:30Z" name: switch-graph-acf44-5486c59c4d-rtpd8.18be7c1dd807ba3d namespace: kserve-ci-e2e-test resourceVersion: "16514" uid: 321d60a1-d2f1-40d6-bfbc-fa698b62805e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:29:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-acf44} kind: Pod name: switch-graph-acf44-5486c59c4d-rtpd8 namespace: kserve-ci-e2e-test resourceVersion: "16357" uid: e68fbe03-b9cf-49d0-8e32-d2d966afc588 kind: Event lastTimestamp: "2026-07-02T13:29:59Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T13:29:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:29:59Z" name: switch-graph-acf44-5486c59c4d-rtpd8.18be7c1ee23ea902 namespace: kserve-ci-e2e-test resourceVersion: "16874" uid: 28171576-7937-48ac-b306-afd8bda5a069 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-122.ec2.internal source: component: kubelet host: ip-10-0-133-122.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:29:19Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-acf44-5486c59c4d namespace: kserve-ci-e2e-test resourceVersion: "16350" uid: cc8dcad0-103a-4336-a660-4b1eefb2b357 kind: Event lastTimestamp: "2026-07-02T13:29:19Z" message: 'Created pod: switch-graph-acf44-5486c59c4d-rtpd8' metadata: creationTimestamp: "2026-07-02T13:29:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:29:19Z" name: switch-graph-acf44-5486c59c4d.18be7c1b745edfd8 namespace: kserve-ci-e2e-test resourceVersion: "16356" uid: c83c0af2-7261-4a9a-874e-f9a522387c6a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:29:19Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-acf44 namespace: kserve-ci-e2e-test resourceVersion: "16348" uid: a6658a68-6b3b-4c1f-afc7-8de3678c2128 kind: Event lastTimestamp: "2026-07-02T13:29:19Z" message: Scaled up replica set switch-graph-acf44-5486c59c4d from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:29:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:29:19Z" name: switch-graph-acf44.18be7c1b7372aefd namespace: kserve-ci-e2e-test resourceVersion: "16351" uid: 018877fc-610d-48c9-92ac-9f839ecfe4e3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:29:19Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-acf44 namespace: kserve-ci-e2e-test resourceVersion: "16365" uid: 20a0437e-64e5-4175-844a-0b4be8d3a97c kind: Event lastTimestamp: "2026-07-02T13:29:19Z" message: InferenceGraph [switch-graph-acf44] is Ready metadata: creationTimestamp: "2026-07-02T13:29:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:29:19Z" name: switch-graph-acf44.18be7c1b77da5263 namespace: kserve-ci-e2e-test resourceVersion: "16367" uid: 9987114a-3628-4379-bf28-8d7d6c2150c3 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:29:19Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-acf44 namespace: kserve-ci-e2e-test resourceVersion: "16343" uid: 20a0437e-64e5-4175-844a-0b4be8d3a97c kind: Event lastTimestamp: "2026-07-02T13:29:19Z" message: 'Failed to update status for InferenceGraph "switch-graph-acf44": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-acf44": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:29:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:29:19Z" name: switch-graph-acf44.18be7c1b79a1bebe namespace: kserve-ci-e2e-test resourceVersion: "16368" uid: 50f11588-14df-4ed6-a7aa-d12f1116eff5 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:29:19Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-acf44 namespace: kserve-ci-e2e-test resourceVersion: "16343" uid: 20a0437e-64e5-4175-844a-0b4be8d3a97c kind: Event lastTimestamp: "2026-07-02T13:29:19Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-acf44": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:29:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:29:19Z" name: switch-graph-acf44.18be7c1b79a21802 namespace: kserve-ci-e2e-test resourceVersion: "16370" uid: a664214a-c9f8-4c0b-81ce-d4dc2865a4f5 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning kind: EventList metadata: resourceVersion: "34897"