--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-04-20T20:26:25.752792Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-38d71-7dbc85bf6b-hh2kv namespace: kserve-ci-e2e-test resourceVersion: "20544" uid: b964d357-86f9-44af-b9b8-8b19415ff4c8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-38d71-7dbc85bf6b-hh2kv to ip-10-0-131-234.ec2.internal metadata: creationTimestamp: "2026-04-20T20:26:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-20T20:26:25Z" name: ensemble-graph-38d71-7dbc85bf6b-hh2kv.18a82a80d145023d namespace: kserve-ci-e2e-test resourceVersion: "20550" uid: 091196c9-22a3-4fa7-b825-f152bc439f15 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58dbb994fc-hbcnb source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:26:25Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-38d71-7dbc85bf6b-hh2kv namespace: kserve-ci-e2e-test resourceVersion: "20547" uid: b964d357-86f9-44af-b9b8-8b19415ff4c8 kind: Event lastTimestamp: "2026-04-20T20:26:26Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-38d71-serving-cert" not found' metadata: creationTimestamp: "2026-04-20T20:26:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:26:26Z" name: ensemble-graph-38d71-7dbc85bf6b-hh2kv.18a82a80dd9dc7b3 namespace: kserve-ci-e2e-test resourceVersion: "20566" uid: 34010f49-6b3d-4a55-b165-0418e8885782 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:26:27Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-38d71-7dbc85bf6b-hh2kv namespace: kserve-ci-e2e-test resourceVersion: "20553" uid: b964d357-86f9-44af-b9b8-8b19415ff4c8 kind: Event lastTimestamp: "2026-04-20T20:26:27Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-20T20:26:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-20T20:26:27Z" name: ensemble-graph-38d71-7dbc85bf6b-hh2kv.18a82a8143f6af3c namespace: kserve-ci-e2e-test resourceVersion: "20575" uid: 8edfa514-2324-431d-949f-6793e25189c0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:26:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-38d71} kind: Pod name: ensemble-graph-38d71-7dbc85bf6b-hh2kv namespace: kserve-ci-e2e-test resourceVersion: "20547" uid: b964d357-86f9-44af-b9b8-8b19415ff4c8 kind: Event lastTimestamp: "2026-04-20T20:26:27Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:0169f8fa51f9c0b83c5c9a121f51af9f3e3cdcc48ff1b00c2fcaa12b38a7914f" already present on machine metadata: creationTimestamp: "2026-04-20T20:26:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:26:27Z" name: ensemble-graph-38d71-7dbc85bf6b-hh2kv.18a82a8145223148 namespace: kserve-ci-e2e-test resourceVersion: "20577" uid: ec006ecd-47b1-43cd-be7c-67a063e3fa7f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:26:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-38d71} kind: Pod name: ensemble-graph-38d71-7dbc85bf6b-hh2kv namespace: kserve-ci-e2e-test resourceVersion: "20547" uid: b964d357-86f9-44af-b9b8-8b19415ff4c8 kind: Event lastTimestamp: "2026-04-20T20:26:27Z" message: 'Created container: ensemble-graph-38d71' metadata: creationTimestamp: "2026-04-20T20:26:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:26:27Z" name: ensemble-graph-38d71-7dbc85bf6b-hh2kv.18a82a81480cd386 namespace: kserve-ci-e2e-test resourceVersion: "20578" uid: 1911da7d-a1ec-49b3-8ece-078fdbdd9f4c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:26:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-38d71} kind: Pod name: ensemble-graph-38d71-7dbc85bf6b-hh2kv namespace: kserve-ci-e2e-test resourceVersion: "20547" uid: b964d357-86f9-44af-b9b8-8b19415ff4c8 kind: Event lastTimestamp: "2026-04-20T20:26:27Z" message: Started container ensemble-graph-38d71 metadata: creationTimestamp: "2026-04-20T20:26:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:26:27Z" name: ensemble-graph-38d71-7dbc85bf6b-hh2kv.18a82a814885b58f namespace: kserve-ci-e2e-test resourceVersion: "20579" uid: febcdd16-9c0e-4f6c-ac17-72a985c1d3ba reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:34:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-38d71} kind: Pod name: ensemble-graph-38d71-7dbc85bf6b-hh2kv namespace: kserve-ci-e2e-test resourceVersion: "20547" uid: b964d357-86f9-44af-b9b8-8b19415ff4c8 kind: Event lastTimestamp: "2026-04-20T20:34:40Z" message: Stopping container ensemble-graph-38d71 metadata: creationTimestamp: "2026-04-20T20:34:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:34:40Z" name: ensemble-graph-38d71-7dbc85bf6b-hh2kv.18a82af3f9c566cf namespace: kserve-ci-e2e-test resourceVersion: "24059" uid: 68a290d2-2eb2-457d-a2a2-6b8b0212f422 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-20T20:34:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-38d71} kind: Pod name: ensemble-graph-38d71-7dbc85bf6b-hh2kv namespace: kserve-ci-e2e-test resourceVersion: "20547" uid: b964d357-86f9-44af-b9b8-8b19415ff4c8 kind: Event lastTimestamp: "2026-04-20T20:35:09Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-20T20:34:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:35:09Z" name: ensemble-graph-38d71-7dbc85bf6b-hh2kv.18a82af4e7788fb7 namespace: kserve-ci-e2e-test resourceVersion: "24405" uid: 80a90ba6-d06b-455b-b960-5afa6ca39c9f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:26:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-38d71-7dbc85bf6b namespace: kserve-ci-e2e-test resourceVersion: "20541" uid: 212816cd-7503-4af1-bc28-ac79bb3f9d7e kind: Event lastTimestamp: "2026-04-20T20:26:25Z" message: 'Created pod: ensemble-graph-38d71-7dbc85bf6b-hh2kv' metadata: creationTimestamp: "2026-04-20T20:26:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:26:25Z" name: ensemble-graph-38d71-7dbc85bf6b.18a82a80d0d8cbdd namespace: kserve-ci-e2e-test resourceVersion: "20545" uid: a94cdf21-8af9-4190-b2af-1eda1c79b57f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:26:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-38d71 namespace: kserve-ci-e2e-test resourceVersion: "20539" uid: 3d159138-ed82-4066-ad46-2bd5e2e87f26 kind: Event lastTimestamp: "2026-04-20T20:26:25Z" message: Scaled up replica set ensemble-graph-38d71-7dbc85bf6b from 0 to 1 metadata: creationTimestamp: "2026-04-20T20:26:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:26:25Z" name: ensemble-graph-38d71.18a82a80d0494022 namespace: kserve-ci-e2e-test resourceVersion: "20542" uid: c6209f6e-17dc-4d44-9053-203fcb860e26 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:26:25Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-38d71 namespace: kserve-ci-e2e-test resourceVersion: "20556" uid: 781ea977-6b53-4eaf-8606-7f0ce5770998 kind: Event lastTimestamp: "2026-04-20T20:26:25Z" message: InferenceGraph [ensemble-graph-38d71] is Ready metadata: creationTimestamp: "2026-04-20T20:26:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:26:25Z" name: ensemble-graph-38d71.18a82a80d3be50a9 namespace: kserve-ci-e2e-test resourceVersion: "20559" uid: 9fb19444-82ac-4bde-816d-fc263bb199a6 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-20T20:26:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-38d71 namespace: kserve-ci-e2e-test resourceVersion: "20540" uid: a5dbb904-c33a-49cc-9c13-807ca6194e75 kind: Event lastTimestamp: "2026-04-20T20:27:10Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:26:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:27:10Z" name: ensemble-graph-38d71.18a82a844ec0b3d9 namespace: kserve-ci-e2e-test resourceVersion: "20974" uid: 1bcb21fd-b92e-414d-85a1-9ecd4e15b50a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-20T20:26:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-38d71 namespace: kserve-ci-e2e-test resourceVersion: "20540" uid: a5dbb904-c33a-49cc-9c13-807ca6194e75 kind: Event lastTimestamp: "2026-04-20T20:27:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:26:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:27:10Z" name: ensemble-graph-38d71.18a82a844f3b8607 namespace: kserve-ci-e2e-test resourceVersion: "20975" uid: 8a9c449e-1b6f-4ae1-b6b4-e934395d065b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-20T20:25:15.443334Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-7541f-6b5b9965d4-lj2ph namespace: kserve-ci-e2e-test resourceVersion: "19584" uid: 381ce4b9-2e11-4621-b3fa-65b4ff7e895f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-7541f-6b5b9965d4-lj2ph to ip-10-0-131-234.ec2.internal metadata: creationTimestamp: "2026-04-20T20:25:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-20T20:25:15Z" name: ensemble-graph-7541f-6b5b9965d4-lj2ph.18a82a70727fcaa7 namespace: kserve-ci-e2e-test resourceVersion: "19589" uid: 4139963e-b05f-4bed-b0ff-61274694081b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58dbb994fc-hbcnb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:15Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-7541f-6b5b9965d4-lj2ph namespace: kserve-ci-e2e-test resourceVersion: "19586" uid: 381ce4b9-2e11-4621-b3fa-65b4ff7e895f kind: Event lastTimestamp: "2026-04-20T20:25:15Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-7541f-serving-cert" not found' metadata: creationTimestamp: "2026-04-20T20:25:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:25:15Z" name: ensemble-graph-7541f-6b5b9965d4-lj2ph.18a82a707a7b7f3d namespace: kserve-ci-e2e-test resourceVersion: "19601" uid: 4efc17d4-5a78-4b19-b22d-6875e1367cf9 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:16Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-7541f-6b5b9965d4-lj2ph namespace: kserve-ci-e2e-test resourceVersion: "19593" uid: 381ce4b9-2e11-4621-b3fa-65b4ff7e895f kind: Event lastTimestamp: "2026-04-20T20:25:16Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-20T20:25:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-20T20:25:16Z" name: ensemble-graph-7541f-6b5b9965d4-lj2ph.18a82a70af684982 namespace: kserve-ci-e2e-test resourceVersion: "19611" uid: f7edc8d9-eeaa-49e8-909b-a89e6143d997 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-7541f} kind: Pod name: ensemble-graph-7541f-6b5b9965d4-lj2ph namespace: kserve-ci-e2e-test resourceVersion: "19586" uid: 381ce4b9-2e11-4621-b3fa-65b4ff7e895f kind: Event lastTimestamp: "2026-04-20T20:25:16Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:0169f8fa51f9c0b83c5c9a121f51af9f3e3cdcc48ff1b00c2fcaa12b38a7914f" already present on machine metadata: creationTimestamp: "2026-04-20T20:25:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:25:16Z" name: ensemble-graph-7541f-6b5b9965d4-lj2ph.18a82a70b09d990c namespace: kserve-ci-e2e-test resourceVersion: "19613" uid: 3da3cc32-d69a-4475-9de1-819ac0fedd18 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-7541f} kind: Pod name: ensemble-graph-7541f-6b5b9965d4-lj2ph namespace: kserve-ci-e2e-test resourceVersion: "19586" uid: 381ce4b9-2e11-4621-b3fa-65b4ff7e895f kind: Event lastTimestamp: "2026-04-20T20:25:16Z" message: 'Created container: ensemble-graph-7541f' metadata: creationTimestamp: "2026-04-20T20:25:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:25:16Z" name: ensemble-graph-7541f-6b5b9965d4-lj2ph.18a82a70b37a7672 namespace: kserve-ci-e2e-test resourceVersion: "19614" uid: 1e66b87a-7959-4d54-81bb-3b01a18eb6d6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-7541f} kind: Pod name: ensemble-graph-7541f-6b5b9965d4-lj2ph namespace: kserve-ci-e2e-test resourceVersion: "19586" uid: 381ce4b9-2e11-4621-b3fa-65b4ff7e895f kind: Event lastTimestamp: "2026-04-20T20:25:16Z" message: Started container ensemble-graph-7541f metadata: creationTimestamp: "2026-04-20T20:25:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:25:16Z" name: ensemble-graph-7541f-6b5b9965d4-lj2ph.18a82a70b3fe5d0d namespace: kserve-ci-e2e-test resourceVersion: "19615" uid: cc319221-f9a6-4966-9625-b78c5f834fc5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-7541f} kind: Pod name: ensemble-graph-7541f-6b5b9965d4-lj2ph namespace: kserve-ci-e2e-test resourceVersion: "19586" uid: 381ce4b9-2e11-4621-b3fa-65b4ff7e895f kind: Event lastTimestamp: "2026-04-20T20:25:25Z" message: Stopping container ensemble-graph-7541f metadata: creationTimestamp: "2026-04-20T20:25:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:25:25Z" name: ensemble-graph-7541f-6b5b9965d4-lj2ph.18a82a72cc86c389 namespace: kserve-ci-e2e-test resourceVersion: "19711" uid: ee85ac40-2248-46c3-9f24-ebc89d73a601 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-20T20:25:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-7541f} kind: Pod name: ensemble-graph-7541f-6b5b9965d4-lj2ph namespace: kserve-ci-e2e-test resourceVersion: "19586" uid: 381ce4b9-2e11-4621-b3fa-65b4ff7e895f kind: Event lastTimestamp: "2026-04-20T20:25:53Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-20T20:25:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:25:53Z" name: ensemble-graph-7541f-6b5b9965d4-lj2ph.18a82a73664d19a8 namespace: kserve-ci-e2e-test resourceVersion: "20065" uid: 78aebd38-9dd0-43e6-8cbe-0dd1e980d1e2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-7541f-6b5b9965d4 namespace: kserve-ci-e2e-test resourceVersion: "19581" uid: 80c16025-6a5a-4ff5-b0cb-2b6410620202 kind: Event lastTimestamp: "2026-04-20T20:25:15Z" message: 'Created pod: ensemble-graph-7541f-6b5b9965d4-lj2ph' metadata: creationTimestamp: "2026-04-20T20:25:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:25:15Z" name: ensemble-graph-7541f-6b5b9965d4.18a82a707223e4d4 namespace: kserve-ci-e2e-test resourceVersion: "19585" uid: 1233504c-5772-41f1-93ff-a20f9287e9ad reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-7541f namespace: kserve-ci-e2e-test resourceVersion: "19579" uid: 7f1a420d-a205-48ce-898d-ec668777268c kind: Event lastTimestamp: "2026-04-20T20:25:15Z" message: Scaled up replica set ensemble-graph-7541f-6b5b9965d4 from 0 to 1 metadata: creationTimestamp: "2026-04-20T20:25:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:25:15Z" name: ensemble-graph-7541f.18a82a70718f2acc namespace: kserve-ci-e2e-test resourceVersion: "19582" uid: 046ec800-e721-40b7-a468-5f2a59104145 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:15Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-7541f namespace: kserve-ci-e2e-test resourceVersion: "19595" uid: 2559523e-0e79-44aa-9bab-fe445ed169ac kind: Event lastTimestamp: "2026-04-20T20:25:15Z" message: InferenceGraph [ensemble-graph-7541f] is Ready metadata: creationTimestamp: "2026-04-20T20:25:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:25:15Z" name: ensemble-graph-7541f.18a82a707515ac09 namespace: kserve-ci-e2e-test resourceVersion: "19597" uid: ebf16056-9462-45a5-ba4c-c0abbe5f352a reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-20T20:13:30.409886Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-127f1-predictor-548d5f46c9-cb6dd namespace: kserve-ci-e2e-test resourceVersion: "13447" uid: f42e68c8-47bf-47a6-8400-a14c37e81640 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-127f1-predictor-548d5f46c9-cb6dd to ip-10-0-130-227.ec2.internal metadata: creationTimestamp: "2026-04-20T20:13:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-20T20:13:30Z" name: error-404-isvc-127f1-predictor-548d5f46c9-cb6dd.18a829cc4b3b1fdb namespace: kserve-ci-e2e-test resourceVersion: "13451" uid: 1766fa69-8ec2-45aa-be0e-8d29d377fdd5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58dbb994fc-hbcnb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:13:31Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-127f1-predictor-548d5f46c9-cb6dd namespace: kserve-ci-e2e-test resourceVersion: "13455" uid: f42e68c8-47bf-47a6-8400-a14c37e81640 kind: Event lastTimestamp: "2026-04-20T20:13:31Z" message: Add eth0 [10.133.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-20T20:13:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-20T20:13:31Z" name: error-404-isvc-127f1-predictor-548d5f46c9-cb6dd.18a829cc70b756e9 namespace: kserve-ci-e2e-test resourceVersion: "13488" uid: 919c7a96-c7df-410d-94ab-f5bfeee43680 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:13:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-127f1-predictor-548d5f46c9-cb6dd namespace: kserve-ci-e2e-test resourceVersion: "13449" uid: f42e68c8-47bf-47a6-8400-a14c37e81640 kind: Event lastTimestamp: "2026-04-20T20:13:31Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1293" metadata: creationTimestamp: "2026-04-20T20:13:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:13:31Z" name: error-404-isvc-127f1-predictor-548d5f46c9-cb6dd.18a829cc72019866 namespace: kserve-ci-e2e-test resourceVersion: "13490" uid: 630d1dde-9a3f-4a5b-9069-e03600473b7a reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:13:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-127f1-predictor-548d5f46c9-cb6dd namespace: kserve-ci-e2e-test resourceVersion: "13449" uid: f42e68c8-47bf-47a6-8400-a14c37e81640 kind: Event lastTimestamp: "2026-04-20T20:13:44Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1293" in 13.431s (13.431s including waiting). Image size: 1332277345 bytes.' metadata: creationTimestamp: "2026-04-20T20:13:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:13:44Z" name: error-404-isvc-127f1-predictor-548d5f46c9-cb6dd.18a829cf9290ad26 namespace: kserve-ci-e2e-test resourceVersion: "13630" uid: 384a2630-d747-47aa-b16b-24c1cd3f9531 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:13:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-127f1-predictor-548d5f46c9-cb6dd namespace: kserve-ci-e2e-test resourceVersion: "13449" uid: f42e68c8-47bf-47a6-8400-a14c37e81640 kind: Event lastTimestamp: "2026-04-20T20:13:44Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-20T20:13:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:13:44Z" name: error-404-isvc-127f1-predictor-548d5f46c9-cb6dd.18a829cf95f102d5 namespace: kserve-ci-e2e-test resourceVersion: "13631" uid: f9fba67a-0dd0-4238-a36f-199ece404d1d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:13:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-127f1-predictor-548d5f46c9-cb6dd namespace: kserve-ci-e2e-test resourceVersion: "13449" uid: f42e68c8-47bf-47a6-8400-a14c37e81640 kind: Event lastTimestamp: "2026-04-20T20:13:44Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-20T20:13:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:13:44Z" name: error-404-isvc-127f1-predictor-548d5f46c9-cb6dd.18a829cf965ed300 namespace: kserve-ci-e2e-test resourceVersion: "13632" uid: 673548df-175a-4cc1-bf0f-b68e760182ca reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:13:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-127f1-predictor-548d5f46c9-cb6dd namespace: kserve-ci-e2e-test resourceVersion: "13449" uid: f42e68c8-47bf-47a6-8400-a14c37e81640 kind: Event lastTimestamp: "2026-04-20T20:13:44Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-04-20T20:13:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:13:44Z" name: error-404-isvc-127f1-predictor-548d5f46c9-cb6dd.18a829cf966abc53 namespace: kserve-ci-e2e-test resourceVersion: "13633" uid: 86b50315-0741-4a87-87de-967a39faee40 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:13:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-127f1-predictor-548d5f46c9-cb6dd namespace: kserve-ci-e2e-test resourceVersion: "13449" uid: f42e68c8-47bf-47a6-8400-a14c37e81640 kind: Event lastTimestamp: "2026-04-20T20:13:47Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.9s (2.9s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-04-20T20:13:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:13:47Z" name: error-404-isvc-127f1-predictor-548d5f46c9-cb6dd.18a829d04350f2cb namespace: kserve-ci-e2e-test resourceVersion: "13689" uid: 617c25bb-4a95-430f-8595-d36111b0ac0d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:13:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-127f1-predictor-548d5f46c9-cb6dd namespace: kserve-ci-e2e-test resourceVersion: "13449" uid: f42e68c8-47bf-47a6-8400-a14c37e81640 kind: Event lastTimestamp: "2026-04-20T20:13:47Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-20T20:13:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:13:47Z" name: error-404-isvc-127f1-predictor-548d5f46c9-cb6dd.18a829d0478d0a38 namespace: kserve-ci-e2e-test resourceVersion: "13693" uid: 2755d61a-fd9c-434b-b9e9-9484c8c1c1c4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:13:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-127f1-predictor-548d5f46c9-cb6dd namespace: kserve-ci-e2e-test resourceVersion: "13449" uid: f42e68c8-47bf-47a6-8400-a14c37e81640 kind: Event lastTimestamp: "2026-04-20T20:13:47Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:13:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:13:47Z" name: error-404-isvc-127f1-predictor-548d5f46c9-cb6dd.18a829d0480cc6e7 namespace: kserve-ci-e2e-test resourceVersion: "13694" uid: 2330021f-9b6b-4efa-a109-c080fc4896a7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-20T20:13:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-127f1-predictor-548d5f46c9-cb6dd namespace: kserve-ci-e2e-test resourceVersion: "13449" uid: f42e68c8-47bf-47a6-8400-a14c37e81640 kind: Event lastTimestamp: "2026-04-20T20:14:24Z" message: 'Readiness probe failed: dial tcp 10.133.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-20T20:13:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:14:24Z" name: error-404-isvc-127f1-predictor-548d5f46c9-cb6dd.18a829d09525c4b8 namespace: kserve-ci-e2e-test resourceVersion: "14045" uid: 6ce3b3bb-2fe9-4193-be20-b42327a37a6c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-127f1-predictor-548d5f46c9-cb6dd namespace: kserve-ci-e2e-test resourceVersion: "13449" uid: f42e68c8-47bf-47a6-8400-a14c37e81640 kind: Event lastTimestamp: "2026-04-20T20:15:00Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-20T20:15:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:15:00Z" name: error-404-isvc-127f1-predictor-548d5f46c9-cb6dd.18a829e1361630cb namespace: kserve-ci-e2e-test resourceVersion: "14419" uid: a6f23f08-849c-4479-9d5c-7fe9a45c3cdd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-127f1-predictor-548d5f46c9-cb6dd namespace: kserve-ci-e2e-test resourceVersion: "13449" uid: f42e68c8-47bf-47a6-8400-a14c37e81640 kind: Event lastTimestamp: "2026-04-20T20:15:00Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:15:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:15:00Z" name: error-404-isvc-127f1-predictor-548d5f46c9-cb6dd.18a829e136173f61 namespace: kserve-ci-e2e-test resourceVersion: "14422" uid: 72d7f2c2-943f-4f79-ba85-c1ac306de105 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:13:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-127f1-predictor-548d5f46c9 namespace: kserve-ci-e2e-test resourceVersion: "13440" uid: e65f6793-62e8-460f-8af9-1f65efe9f3df kind: Event lastTimestamp: "2026-04-20T20:13:30Z" message: 'Created pod: error-404-isvc-127f1-predictor-548d5f46c9-cb6dd' metadata: creationTimestamp: "2026-04-20T20:13:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:13:30Z" name: error-404-isvc-127f1-predictor-548d5f46c9.18a829cc4ac8d52a namespace: kserve-ci-e2e-test resourceVersion: "13448" uid: a9b44f06-b16f-413b-ad1e-7cb59136f272 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:13:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-127f1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13438" uid: d5422979-d192-4579-ace3-31d1a5f61001 kind: Event lastTimestamp: "2026-04-20T20:13:30Z" message: Scaled up replica set error-404-isvc-127f1-predictor-548d5f46c9 from 0 to 1 metadata: creationTimestamp: "2026-04-20T20:13:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:13:30Z" name: error-404-isvc-127f1-predictor.18a829cc45df54dc namespace: kserve-ci-e2e-test resourceVersion: "13441" uid: 14742609-f1e7-4367-a405-bf3e63972a9d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-20T20:13:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-127f1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13439" uid: dd91185e-5604-4a9c-a8bf-74fe8d3e32e9 kind: Event lastTimestamp: "2026-04-20T20:14:15Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:13:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:14:15Z" name: error-404-isvc-127f1-predictor.18a829cfc45e9292 namespace: kserve-ci-e2e-test resourceVersion: "13970" uid: a0715b00-62ce-406b-9416-78b276d3da8b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-20T20:13:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-127f1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13439" uid: dd91185e-5604-4a9c-a8bf-74fe8d3e32e9 kind: Event lastTimestamp: "2026-04-20T20:14:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:13:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:14:15Z" name: error-404-isvc-127f1-predictor.18a829cfc4c6371f namespace: kserve-ci-e2e-test resourceVersion: "13971" uid: 90ad0928-c2a1-40a4-912d-e59c398ed0ef reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:14:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-127f1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13662" uid: dd91185e-5604-4a9c-a8bf-74fe8d3e32e9 kind: Event lastTimestamp: "2026-04-20T20:14:45Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:14:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:14:45Z" name: error-404-isvc-127f1-predictor.18a829da40f14215 namespace: kserve-ci-e2e-test resourceVersion: "14224" uid: 36e42a7c-b2fc-4630-9552-ce4fd2e7ed88 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:14:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-127f1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13662" uid: dd91185e-5604-4a9c-a8bf-74fe8d3e32e9 kind: Event lastTimestamp: "2026-04-20T20:14:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:14:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:14:45Z" name: error-404-isvc-127f1-predictor.18a829da41555030 namespace: kserve-ci-e2e-test resourceVersion: "14225" uid: c199a97f-ca24-43e4-af5f-d5fe9845f7e7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-20T20:13:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-127f1 namespace: kserve-ci-e2e-test resourceVersion: "13422" uid: ea6c3e1a-b58e-4f17-850f-d36f27885c54 kind: Event lastTimestamp: "2026-04-20T20:14:34Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-20T20:13:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:14:34Z" name: error-404-isvc-127f1.18a829cc45b8daf7 namespace: kserve-ci-e2e-test resourceVersion: "14145" uid: d4364bfa-1dbb-48ef-be49-0ecec60874db reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:14:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-127f1 namespace: kserve-ci-e2e-test resourceVersion: "14143" uid: ea6c3e1a-b58e-4f17-850f-d36f27885c54 kind: Event lastTimestamp: "2026-04-20T20:14:34Z" message: InferenceService [error-404-isvc-127f1] is Ready metadata: creationTimestamp: "2026-04-20T20:14:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:14:34Z" name: error-404-isvc-127f1.18a829db5001029d namespace: kserve-ci-e2e-test resourceVersion: "14144" uid: 959530cf-62cc-46d1-90c7-d427fd8bc66a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-20T20:25:25.765062Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-38d71-predictor-5cd74bff75-4bl6s namespace: kserve-ci-e2e-test resourceVersion: "19787" uid: 4864b38c-ba48-4aa5-aabb-7a5a221ab350 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-38d71-predictor-5cd74bff75-4bl6s to ip-10-0-130-227.ec2.internal metadata: creationTimestamp: "2026-04-20T20:25:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-20T20:25:25Z" name: error-404-isvc-38d71-predictor-5cd74bff75-4bl6s.18a82a72d9b8d293 namespace: kserve-ci-e2e-test resourceVersion: "19791" uid: a2dca69a-8656-4f92-99ea-72e8f895e2d9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58dbb994fc-hbcnb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:26Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-38d71-predictor-5cd74bff75-4bl6s namespace: kserve-ci-e2e-test resourceVersion: "19796" uid: 4864b38c-ba48-4aa5-aabb-7a5a221ab350 kind: Event lastTimestamp: "2026-04-20T20:25:26Z" message: Add eth0 [10.133.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-20T20:25:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-20T20:25:26Z" name: error-404-isvc-38d71-predictor-5cd74bff75-4bl6s.18a82a72f359a949 namespace: kserve-ci-e2e-test resourceVersion: "19808" uid: 94029892-d19e-4a68-85cc-7f7f3bd3f42c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-38d71-predictor-5cd74bff75-4bl6s namespace: kserve-ci-e2e-test resourceVersion: "19789" uid: 4864b38c-ba48-4aa5-aabb-7a5a221ab350 kind: Event lastTimestamp: "2026-04-20T20:25:26Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-20T20:25:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:25:26Z" name: error-404-isvc-38d71-predictor-5cd74bff75-4bl6s.18a82a72f4955a86 namespace: kserve-ci-e2e-test resourceVersion: "19810" uid: 87965c74-a1f9-4550-a1b6-3fb119580a7c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-38d71-predictor-5cd74bff75-4bl6s namespace: kserve-ci-e2e-test resourceVersion: "19789" uid: 4864b38c-ba48-4aa5-aabb-7a5a221ab350 kind: Event lastTimestamp: "2026-04-20T20:25:26Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-20T20:25:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:25:26Z" name: error-404-isvc-38d71-predictor-5cd74bff75-4bl6s.18a82a72f79d79e1 namespace: kserve-ci-e2e-test resourceVersion: "19811" uid: d9ec2192-94df-493f-bd35-358e7ffa49cf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-38d71-predictor-5cd74bff75-4bl6s namespace: kserve-ci-e2e-test resourceVersion: "19789" uid: 4864b38c-ba48-4aa5-aabb-7a5a221ab350 kind: Event lastTimestamp: "2026-04-20T20:25:26Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-20T20:25:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:25:26Z" name: error-404-isvc-38d71-predictor-5cd74bff75-4bl6s.18a82a72f80cfbb6 namespace: kserve-ci-e2e-test resourceVersion: "19812" uid: 687fc7e1-9b75-4e54-bb68-4ea225a23ef0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-38d71-predictor-5cd74bff75-4bl6s namespace: kserve-ci-e2e-test resourceVersion: "19789" uid: 4864b38c-ba48-4aa5-aabb-7a5a221ab350 kind: Event lastTimestamp: "2026-04-20T20:25:26Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-20T20:25:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:25:26Z" name: error-404-isvc-38d71-predictor-5cd74bff75-4bl6s.18a82a72f818b870 namespace: kserve-ci-e2e-test resourceVersion: "19813" uid: 22c31aff-1ec8-4a7c-9e58-4975d6098b0d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-38d71-predictor-5cd74bff75-4bl6s namespace: kserve-ci-e2e-test resourceVersion: "19789" uid: 4864b38c-ba48-4aa5-aabb-7a5a221ab350 kind: Event lastTimestamp: "2026-04-20T20:25:26Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-20T20:25:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:25:26Z" name: error-404-isvc-38d71-predictor-5cd74bff75-4bl6s.18a82a72fad13679 namespace: kserve-ci-e2e-test resourceVersion: "19814" uid: 2967ebee-3160-455d-b9e2-f8e512db37ea reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-38d71-predictor-5cd74bff75-4bl6s namespace: kserve-ci-e2e-test resourceVersion: "19789" uid: 4864b38c-ba48-4aa5-aabb-7a5a221ab350 kind: Event lastTimestamp: "2026-04-20T20:25:26Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:25:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:25:26Z" name: error-404-isvc-38d71-predictor-5cd74bff75-4bl6s.18a82a72fb3f52d3 namespace: kserve-ci-e2e-test resourceVersion: "19815" uid: d383d18b-84a9-4261-a9da-548864e6f3d6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-20T20:25:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-38d71-predictor-5cd74bff75-4bl6s namespace: kserve-ci-e2e-test resourceVersion: "19789" uid: 4864b38c-ba48-4aa5-aabb-7a5a221ab350 kind: Event lastTimestamp: "2026-04-20T20:34:43Z" message: 'Readiness probe failed: dial tcp 10.133.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-20T20:25:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:34:43Z" name: error-404-isvc-38d71-predictor-5cd74bff75-4bl6s.18a82a732f746101 namespace: kserve-ci-e2e-test resourceVersion: "24201" uid: 7e2a51ac-be35-49c4-8e42-f3afc5f3c822 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:34:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-38d71-predictor-5cd74bff75-4bl6s namespace: kserve-ci-e2e-test resourceVersion: "19789" uid: 4864b38c-ba48-4aa5-aabb-7a5a221ab350 kind: Event lastTimestamp: "2026-04-20T20:34:40Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-20T20:34:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:34:40Z" name: error-404-isvc-38d71-predictor-5cd74bff75-4bl6s.18a82af403e7b489 namespace: kserve-ci-e2e-test resourceVersion: "24109" uid: 8e7bf0f4-6601-449c-a885-cb88e6c6f4d4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:34:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-38d71-predictor-5cd74bff75-4bl6s namespace: kserve-ci-e2e-test resourceVersion: "19789" uid: 4864b38c-ba48-4aa5-aabb-7a5a221ab350 kind: Event lastTimestamp: "2026-04-20T20:34:40Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:34:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:34:40Z" name: error-404-isvc-38d71-predictor-5cd74bff75-4bl6s.18a82af403e8fa92 namespace: kserve-ci-e2e-test resourceVersion: "24112" uid: e353dfae-e3bb-4caf-ad8d-00841842943b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:34:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-38d71-predictor-5cd74bff75-4bl6s namespace: kserve-ci-e2e-test resourceVersion: "19789" uid: 4864b38c-ba48-4aa5-aabb-7a5a221ab350 kind: Event lastTimestamp: "2026-04-20T20:34:43Z" message: 'Readiness probe failed: Get "https://10.133.0.38:8643/healthz": dial tcp 10.133.0.38:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-20T20:34:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:34:43Z" name: error-404-isvc-38d71-predictor-5cd74bff75-4bl6s.18a82af4a3cec242 namespace: kserve-ci-e2e-test resourceVersion: "24200" uid: 12ab15d5-cf34-43ad-bca1-bd56c54598de reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-38d71-predictor-5cd74bff75 namespace: kserve-ci-e2e-test resourceVersion: "19780" uid: 13d6a137-9a21-456d-946d-b9809a816fbf kind: Event lastTimestamp: "2026-04-20T20:25:25Z" message: 'Created pod: error-404-isvc-38d71-predictor-5cd74bff75-4bl6s' metadata: creationTimestamp: "2026-04-20T20:25:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:25:25Z" name: error-404-isvc-38d71-predictor-5cd74bff75.18a82a72d92e95d7 namespace: kserve-ci-e2e-test resourceVersion: "19788" uid: f26e6ef7-b24f-4518-8196-440ff01711fc reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-38d71-predictor namespace: kserve-ci-e2e-test resourceVersion: "19776" uid: 46200223-6ede-4f45-a509-1146735e60bf kind: Event lastTimestamp: "2026-04-20T20:25:25Z" message: Scaled up replica set error-404-isvc-38d71-predictor-5cd74bff75 from 0 to 1 metadata: creationTimestamp: "2026-04-20T20:25:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:25:25Z" name: error-404-isvc-38d71-predictor.18a82a72d7a92e1f namespace: kserve-ci-e2e-test resourceVersion: "19782" uid: 266363df-bbce-46cc-868b-f7fc8a62fa5e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:25:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-38d71-predictor namespace: kserve-ci-e2e-test resourceVersion: "19779" uid: 6876c3cc-b919-48b5-90ca-4a371f69e7a4 kind: Event lastTimestamp: "2026-04-20T20:25:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:25:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:25:55Z" name: error-404-isvc-38d71-predictor.18a82a76564d9270 namespace: kserve-ci-e2e-test resourceVersion: "20127" uid: 27ae34d9-a01a-4b3c-bc6f-1fa4b20c3b38 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:25:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-38d71-predictor namespace: kserve-ci-e2e-test resourceVersion: "19779" uid: 6876c3cc-b919-48b5-90ca-4a371f69e7a4 kind: Event lastTimestamp: "2026-04-20T20:25:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:25:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:25:55Z" name: error-404-isvc-38d71-predictor.18a82a7656b7b044 namespace: kserve-ci-e2e-test resourceVersion: "20128" uid: f4cac8eb-9e55-46ae-b1b5-4da55fe60137 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-20T20:26:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-38d71-predictor namespace: kserve-ci-e2e-test resourceVersion: "19958" uid: 6876c3cc-b919-48b5-90ca-4a371f69e7a4 kind: Event lastTimestamp: "2026-04-20T20:26:40Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:26:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:26:40Z" name: error-404-isvc-38d71-predictor.18a82a7d541031f2 namespace: kserve-ci-e2e-test resourceVersion: "20700" uid: d2dac28d-095d-426f-ac27-f382a8f1a6f1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-20T20:26:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-38d71-predictor namespace: kserve-ci-e2e-test resourceVersion: "19958" uid: 6876c3cc-b919-48b5-90ca-4a371f69e7a4 kind: Event lastTimestamp: "2026-04-20T20:26:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:26:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:26:40Z" name: error-404-isvc-38d71-predictor.18a82a7d5474a364 namespace: kserve-ci-e2e-test resourceVersion: "20701" uid: 64ec6c9c-c78a-4e1a-975b-427b5e06ac29 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-20T20:25:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-38d71 namespace: kserve-ci-e2e-test resourceVersion: "19747" uid: 8c5732a9-7244-4ff7-aa5e-a020aeb1ceaf kind: Event lastTimestamp: "2026-04-20T20:26:13Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-20T20:25:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:26:13Z" name: error-404-isvc-38d71.18a82a72d78a8515 namespace: kserve-ci-e2e-test resourceVersion: "20425" uid: abe6e044-2fb3-45c3-b72d-06923f6c7cd0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-38d71 namespace: kserve-ci-e2e-test resourceVersion: "19747" uid: 8c5732a9-7244-4ff7-aa5e-a020aeb1ceaf kind: Event lastTimestamp: "2026-04-20T20:25:25Z" message: 'Failed to update status for InferenceService "error-404-isvc-38d71": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-38d71": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-20T20:25:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:25:25Z" name: error-404-isvc-38d71.18a82a72d8238d08 namespace: kserve-ci-e2e-test resourceVersion: "19784" uid: 07b0d7a8-f10d-4b38-894c-aa06a3281933 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-38d71 namespace: kserve-ci-e2e-test resourceVersion: "19747" uid: 8c5732a9-7244-4ff7-aa5e-a020aeb1ceaf kind: Event lastTimestamp: "2026-04-20T20:25:25Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-38d71": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-20T20:25:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:25:25Z" name: error-404-isvc-38d71.18a82a72d823cd54 namespace: kserve-ci-e2e-test resourceVersion: "19786" uid: 8257b62c-f199-45ff-ad32-37242704920e reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:26:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-38d71 namespace: kserve-ci-e2e-test resourceVersion: "20423" uid: 8c5732a9-7244-4ff7-aa5e-a020aeb1ceaf kind: Event lastTimestamp: "2026-04-20T20:26:13Z" message: InferenceService [error-404-isvc-38d71] is Ready metadata: creationTimestamp: "2026-04-20T20:26:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:26:13Z" name: error-404-isvc-38d71.18a82a7de9ecebe2 namespace: kserve-ci-e2e-test resourceVersion: "20424" uid: c56f83c8-a2d6-4e38-88b7-25e778f8dbbe reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-20T20:35:50.918144Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-57557-predictor-7b7cb58b6c-hbbsv namespace: kserve-ci-e2e-test resourceVersion: "25033" uid: 3e37579e-e9e4-4880-b2c5-355442b475b1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-57557-predictor-7b7cb58b6c-hbbsv to ip-10-0-130-227.ec2.internal metadata: creationTimestamp: "2026-04-20T20:35:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-20T20:35:50Z" name: error-404-isvc-57557-predictor-7b7cb58b6c-hbbsv.18a82b0467bfde6a namespace: kserve-ci-e2e-test resourceVersion: "25037" uid: b2725840-5288-4c91-9bdd-8198171846bf reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58dbb994fc-hbcnb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:51Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-57557-predictor-7b7cb58b6c-hbbsv namespace: kserve-ci-e2e-test resourceVersion: "25043" uid: 3e37579e-e9e4-4880-b2c5-355442b475b1 kind: Event lastTimestamp: "2026-04-20T20:35:51Z" message: Add eth0 [10.133.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-20T20:35:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-20T20:35:51Z" name: error-404-isvc-57557-predictor-7b7cb58b6c-hbbsv.18a82b04810febfd namespace: kserve-ci-e2e-test resourceVersion: "25053" uid: 95448908-2032-4c09-b572-986d9674103b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-57557-predictor-7b7cb58b6c-hbbsv namespace: kserve-ci-e2e-test resourceVersion: "25035" uid: 3e37579e-e9e4-4880-b2c5-355442b475b1 kind: Event lastTimestamp: "2026-04-20T20:35:51Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-20T20:35:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:35:51Z" name: error-404-isvc-57557-predictor-7b7cb58b6c-hbbsv.18a82b048251483c namespace: kserve-ci-e2e-test resourceVersion: "25055" uid: 9ea87556-630c-445b-8512-a1a39efe0a88 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-57557-predictor-7b7cb58b6c-hbbsv namespace: kserve-ci-e2e-test resourceVersion: "25035" uid: 3e37579e-e9e4-4880-b2c5-355442b475b1 kind: Event lastTimestamp: "2026-04-20T20:35:51Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-20T20:35:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:35:51Z" name: error-404-isvc-57557-predictor-7b7cb58b6c-hbbsv.18a82b04852b6a55 namespace: kserve-ci-e2e-test resourceVersion: "25058" uid: 3afbed74-ffbb-4eef-a2c7-f52f7a6488a0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-57557-predictor-7b7cb58b6c-hbbsv namespace: kserve-ci-e2e-test resourceVersion: "25035" uid: 3e37579e-e9e4-4880-b2c5-355442b475b1 kind: Event lastTimestamp: "2026-04-20T20:35:51Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-20T20:35:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:35:51Z" name: error-404-isvc-57557-predictor-7b7cb58b6c-hbbsv.18a82b0485a153d1 namespace: kserve-ci-e2e-test resourceVersion: "25059" uid: 6be28f14-b585-49fb-9bab-f1fe77e1933e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-57557-predictor-7b7cb58b6c-hbbsv namespace: kserve-ci-e2e-test resourceVersion: "25035" uid: 3e37579e-e9e4-4880-b2c5-355442b475b1 kind: Event lastTimestamp: "2026-04-20T20:35:51Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-20T20:35:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:35:51Z" name: error-404-isvc-57557-predictor-7b7cb58b6c-hbbsv.18a82b0485ae76fe namespace: kserve-ci-e2e-test resourceVersion: "25060" uid: bbf6f398-275c-4e6f-8348-7ddfec96aa18 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-57557-predictor-7b7cb58b6c-hbbsv namespace: kserve-ci-e2e-test resourceVersion: "25035" uid: 3e37579e-e9e4-4880-b2c5-355442b475b1 kind: Event lastTimestamp: "2026-04-20T20:35:51Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-20T20:35:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:35:51Z" name: error-404-isvc-57557-predictor-7b7cb58b6c-hbbsv.18a82b04888eae2e namespace: kserve-ci-e2e-test resourceVersion: "25061" uid: 0db5a07b-fefa-436e-acf9-a6a7f7f7f9a4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-57557-predictor-7b7cb58b6c-hbbsv namespace: kserve-ci-e2e-test resourceVersion: "25035" uid: 3e37579e-e9e4-4880-b2c5-355442b475b1 kind: Event lastTimestamp: "2026-04-20T20:35:51Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:35:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:35:51Z" name: error-404-isvc-57557-predictor-7b7cb58b6c-hbbsv.18a82b048902047d namespace: kserve-ci-e2e-test resourceVersion: "25062" uid: dc181113-2d1f-4819-b4bf-de1576511ad2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-20T20:35:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-57557-predictor-7b7cb58b6c-hbbsv namespace: kserve-ci-e2e-test resourceVersion: "25035" uid: 3e37579e-e9e4-4880-b2c5-355442b475b1 kind: Event lastTimestamp: "2026-04-20T20:36:29Z" message: 'Readiness probe failed: dial tcp 10.133.0.42:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-20T20:35:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:36:29Z" name: error-404-isvc-57557-predictor-7b7cb58b6c-hbbsv.18a82b04fa00a080 namespace: kserve-ci-e2e-test resourceVersion: "25454" uid: 48ab4396-ca8a-4de7-943c-7e858db12e55 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:45:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-57557-predictor-7b7cb58b6c-hbbsv namespace: kserve-ci-e2e-test resourceVersion: "25035" uid: 3e37579e-e9e4-4880-b2c5-355442b475b1 kind: Event lastTimestamp: "2026-04-20T20:45:05Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-20T20:45:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:45:05Z" name: error-404-isvc-57557-predictor-7b7cb58b6c-hbbsv.18a82b858d8d4f01 namespace: kserve-ci-e2e-test resourceVersion: "29442" uid: ec9b8873-c580-4829-95f5-a6d2af924dfd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:45:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-57557-predictor-7b7cb58b6c-hbbsv namespace: kserve-ci-e2e-test resourceVersion: "25035" uid: 3e37579e-e9e4-4880-b2c5-355442b475b1 kind: Event lastTimestamp: "2026-04-20T20:45:05Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:45:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:45:05Z" name: error-404-isvc-57557-predictor-7b7cb58b6c-hbbsv.18a82b858d8e7297 namespace: kserve-ci-e2e-test resourceVersion: "29443" uid: bd1e508a-333f-4c3d-82db-8643822da04b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-57557-predictor-7b7cb58b6c namespace: kserve-ci-e2e-test resourceVersion: "25026" uid: f3083452-d642-43ef-a34b-403fdefe8812 kind: Event lastTimestamp: "2026-04-20T20:35:50Z" message: 'Created pod: error-404-isvc-57557-predictor-7b7cb58b6c-hbbsv' metadata: creationTimestamp: "2026-04-20T20:35:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:35:50Z" name: error-404-isvc-57557-predictor-7b7cb58b6c.18a82b0467340609 namespace: kserve-ci-e2e-test resourceVersion: "25034" uid: eca9b8d2-9027-4f7c-a528-8f980dd56906 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-57557-predictor namespace: kserve-ci-e2e-test resourceVersion: "25023" uid: e39ad530-da00-4ecd-9012-7325d6f8538d kind: Event lastTimestamp: "2026-04-20T20:35:50Z" message: Scaled up replica set error-404-isvc-57557-predictor-7b7cb58b6c from 0 to 1 metadata: creationTimestamp: "2026-04-20T20:35:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:35:50Z" name: error-404-isvc-57557-predictor.18a82b04659f0249 namespace: kserve-ci-e2e-test resourceVersion: "25028" uid: 9bc5984d-a6a3-4042-ae46-f655e903df2b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-20T20:36:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-57557-predictor namespace: kserve-ci-e2e-test resourceVersion: "25025" uid: ad01e1aa-7059-43c5-aa90-b8903ebd4dcd kind: Event lastTimestamp: "2026-04-20T20:37:05Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:36:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:37:05Z" name: error-404-isvc-57557-predictor.18a82b07e40c67df namespace: kserve-ci-e2e-test resourceVersion: "25785" uid: acf59684-363a-46ba-a57c-388796ec13bf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-20T20:36:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-57557-predictor namespace: kserve-ci-e2e-test resourceVersion: "25025" uid: ad01e1aa-7059-43c5-aa90-b8903ebd4dcd kind: Event lastTimestamp: "2026-04-20T20:37:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:36:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:37:05Z" name: error-404-isvc-57557-predictor.18a82b07e46eb714 namespace: kserve-ci-e2e-test resourceVersion: "25786" uid: e2ef85d4-e7bd-4acb-a49a-4ae2f9006733 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-20T20:35:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-57557 namespace: kserve-ci-e2e-test resourceVersion: "25009" uid: 6e31e5f5-31b0-4104-be7a-ce2fcea44941 kind: Event lastTimestamp: "2026-04-20T20:36:39Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-20T20:35:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:36:39Z" name: error-404-isvc-57557.18a82b04657f70cf namespace: kserve-ci-e2e-test resourceVersion: "25551" uid: f6b34890-7745-4d13-a4a7-1f24440f4b03 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-57557 namespace: kserve-ci-e2e-test resourceVersion: "25009" uid: 6e31e5f5-31b0-4104-be7a-ce2fcea44941 kind: Event lastTimestamp: "2026-04-20T20:35:50Z" message: 'Failed to update status for InferenceService "error-404-isvc-57557": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-57557": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-20T20:35:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:35:50Z" name: error-404-isvc-57557.18a82b0465e904e5 namespace: kserve-ci-e2e-test resourceVersion: "25030" uid: 22417c3c-46c8-4976-8a63-01dfcbc58c62 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-57557 namespace: kserve-ci-e2e-test resourceVersion: "25009" uid: 6e31e5f5-31b0-4104-be7a-ce2fcea44941 kind: Event lastTimestamp: "2026-04-20T20:35:50Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-57557": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-20T20:35:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:35:50Z" name: error-404-isvc-57557.18a82b0465e9546a namespace: kserve-ci-e2e-test resourceVersion: "25031" uid: 4f2ef700-9c90-4f59-9b48-08a34ee4498f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:36:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-57557 namespace: kserve-ci-e2e-test resourceVersion: "25549" uid: 6e31e5f5-31b0-4104-be7a-ce2fcea44941 kind: Event lastTimestamp: "2026-04-20T20:36:39Z" message: InferenceService [error-404-isvc-57557] is Ready metadata: creationTimestamp: "2026-04-20T20:36:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:36:39Z" name: error-404-isvc-57557.18a82b0fb4806b57 namespace: kserve-ci-e2e-test resourceVersion: "25550" uid: 1918cbb6-cefa-4f22-b7e9-d0c6cd4d0d97 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-20T20:35:20.426547Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-60a5c-predictor-74955c648c-g4rt8 namespace: kserve-ci-e2e-test resourceVersion: "24585" uid: d712806c-4d72-49b1-a918-2a2aff1ec86f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-60a5c-predictor-74955c648c-g4rt8 to ip-10-0-130-227.ec2.internal metadata: creationTimestamp: "2026-04-20T20:35:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-20T20:35:20Z" name: error-404-isvc-60a5c-predictor-74955c648c-g4rt8.18a82afd4e4f0f36 namespace: kserve-ci-e2e-test resourceVersion: "24590" uid: f7a01a05-dbd9-4183-9a7a-7f1302be446b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58dbb994fc-hbcnb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:20Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-60a5c-predictor-74955c648c-g4rt8 namespace: kserve-ci-e2e-test resourceVersion: "24586" uid: d712806c-4d72-49b1-a918-2a2aff1ec86f kind: Event lastTimestamp: "2026-04-20T20:35:20Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-60a5c-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-20T20:35:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:35:20Z" name: error-404-isvc-60a5c-predictor-74955c648c-g4rt8.18a82afd5c9382a8 namespace: kserve-ci-e2e-test resourceVersion: "24599" uid: b4039a25-e5df-4b8b-8853-77788adf9cdf reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:21Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-60a5c-predictor-74955c648c-g4rt8 namespace: kserve-ci-e2e-test resourceVersion: "24596" uid: d712806c-4d72-49b1-a918-2a2aff1ec86f kind: Event lastTimestamp: "2026-04-20T20:35:21Z" message: Add eth0 [10.133.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-20T20:35:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-20T20:35:21Z" name: error-404-isvc-60a5c-predictor-74955c648c-g4rt8.18a82afd8bae40a6 namespace: kserve-ci-e2e-test resourceVersion: "24617" uid: 6ae5be40-aed4-4d9a-b2da-a17435f912c4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-60a5c-predictor-74955c648c-g4rt8 namespace: kserve-ci-e2e-test resourceVersion: "24586" uid: d712806c-4d72-49b1-a918-2a2aff1ec86f kind: Event lastTimestamp: "2026-04-20T20:35:21Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-20T20:35:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:35:21Z" name: error-404-isvc-60a5c-predictor-74955c648c-g4rt8.18a82afd8d002ea2 namespace: kserve-ci-e2e-test resourceVersion: "24619" uid: 9420873d-7d0c-4410-acb2-dde78313b294 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-60a5c-predictor-74955c648c-g4rt8 namespace: kserve-ci-e2e-test resourceVersion: "24586" uid: d712806c-4d72-49b1-a918-2a2aff1ec86f kind: Event lastTimestamp: "2026-04-20T20:35:21Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-20T20:35:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:35:21Z" name: error-404-isvc-60a5c-predictor-74955c648c-g4rt8.18a82afd904819b9 namespace: kserve-ci-e2e-test resourceVersion: "24623" uid: 7d2814a8-f03a-4e76-b00e-901c4f4ed72f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-60a5c-predictor-74955c648c-g4rt8 namespace: kserve-ci-e2e-test resourceVersion: "24586" uid: d712806c-4d72-49b1-a918-2a2aff1ec86f kind: Event lastTimestamp: "2026-04-20T20:35:21Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-20T20:35:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:35:21Z" name: error-404-isvc-60a5c-predictor-74955c648c-g4rt8.18a82afd90c00ffe namespace: kserve-ci-e2e-test resourceVersion: "24624" uid: 929684c8-732c-45e3-8dea-b482c1266ffc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-60a5c-predictor-74955c648c-g4rt8 namespace: kserve-ci-e2e-test resourceVersion: "24586" uid: d712806c-4d72-49b1-a918-2a2aff1ec86f kind: Event lastTimestamp: "2026-04-20T20:35:21Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-20T20:35:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:35:21Z" name: error-404-isvc-60a5c-predictor-74955c648c-g4rt8.18a82afd90cd4e47 namespace: kserve-ci-e2e-test resourceVersion: "24625" uid: 996bd0a1-0e9e-4f0b-9418-f4255458f91a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-60a5c-predictor-74955c648c-g4rt8 namespace: kserve-ci-e2e-test resourceVersion: "24586" uid: d712806c-4d72-49b1-a918-2a2aff1ec86f kind: Event lastTimestamp: "2026-04-20T20:35:21Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-20T20:35:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:35:21Z" name: error-404-isvc-60a5c-predictor-74955c648c-g4rt8.18a82afd93aa16fc namespace: kserve-ci-e2e-test resourceVersion: "24626" uid: e4bc5598-f5c1-4eba-bea6-6cd25dd5c72c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-60a5c-predictor-74955c648c-g4rt8 namespace: kserve-ci-e2e-test resourceVersion: "24586" uid: d712806c-4d72-49b1-a918-2a2aff1ec86f kind: Event lastTimestamp: "2026-04-20T20:35:21Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:35:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:35:21Z" name: error-404-isvc-60a5c-predictor-74955c648c-g4rt8.18a82afd941967aa namespace: kserve-ci-e2e-test resourceVersion: "24627" uid: 35dac5b3-d732-4567-8a11-360bd648b8ac reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-20T20:35:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-60a5c-predictor-74955c648c-g4rt8 namespace: kserve-ci-e2e-test resourceVersion: "24586" uid: d712806c-4d72-49b1-a918-2a2aff1ec86f kind: Event lastTimestamp: "2026-04-20T20:35:58Z" message: 'Readiness probe failed: dial tcp 10.133.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-20T20:35:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:35:58Z" name: error-404-isvc-60a5c-predictor-74955c648c-g4rt8.18a82afdbb0aae9e namespace: kserve-ci-e2e-test resourceVersion: "25142" uid: 804623fb-909c-41c6-b911-c597d10d8396 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:52:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-60a5c-predictor-74955c648c-g4rt8 namespace: kserve-ci-e2e-test resourceVersion: "24586" uid: d712806c-4d72-49b1-a918-2a2aff1ec86f kind: Event lastTimestamp: "2026-04-20T20:52:40Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-20T20:52:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:52:40Z" name: error-404-isvc-60a5c-predictor-74955c648c-g4rt8.18a82bef5ee0cdec namespace: kserve-ci-e2e-test resourceVersion: "32599" uid: a0e9c6fc-8c21-4438-a596-57b2de68c54a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:52:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-60a5c-predictor-74955c648c-g4rt8 namespace: kserve-ci-e2e-test resourceVersion: "24586" uid: d712806c-4d72-49b1-a918-2a2aff1ec86f kind: Event lastTimestamp: "2026-04-20T20:52:40Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:52:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:52:40Z" name: error-404-isvc-60a5c-predictor-74955c648c-g4rt8.18a82bef5ee17577 namespace: kserve-ci-e2e-test resourceVersion: "32600" uid: b5a78da8-1e41-408a-b8a3-dbec55011b5e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-60a5c-predictor-74955c648c namespace: kserve-ci-e2e-test resourceVersion: "24577" uid: 6e13f681-1262-4c38-8a63-a23dd9f840d0 kind: Event lastTimestamp: "2026-04-20T20:35:20Z" message: 'Created pod: error-404-isvc-60a5c-predictor-74955c648c-g4rt8' metadata: creationTimestamp: "2026-04-20T20:35:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:35:20Z" name: error-404-isvc-60a5c-predictor-74955c648c.18a82afd4ded8751 namespace: kserve-ci-e2e-test resourceVersion: "24587" uid: 14e30a68-1d64-4381-9b8a-49d60ec51fa5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-60a5c-predictor namespace: kserve-ci-e2e-test resourceVersion: "24574" uid: fc0a1dbd-2b93-47ea-9610-9734ff36f6d2 kind: Event lastTimestamp: "2026-04-20T20:35:20Z" message: Scaled up replica set error-404-isvc-60a5c-predictor-74955c648c from 0 to 1 metadata: creationTimestamp: "2026-04-20T20:35:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:35:20Z" name: error-404-isvc-60a5c-predictor.18a82afd4c426d73 namespace: kserve-ci-e2e-test resourceVersion: "24580" uid: 177b3613-db86-4f0c-9a92-f2b0c7d3097f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:35:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-60a5c-predictor namespace: kserve-ci-e2e-test resourceVersion: "24576" uid: 57f41437-414e-44f8-83b5-72c8a9416fab kind: Event lastTimestamp: "2026-04-20T20:35:50Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:35:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:35:50Z" name: error-404-isvc-60a5c-predictor.18a82b00caae89f4 namespace: kserve-ci-e2e-test resourceVersion: "24934" uid: ce8acce4-6f7f-4012-92e9-d49d69bee7b6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:35:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-60a5c-predictor namespace: kserve-ci-e2e-test resourceVersion: "24576" uid: 57f41437-414e-44f8-83b5-72c8a9416fab kind: Event lastTimestamp: "2026-04-20T20:35:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:35:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:35:50Z" name: error-404-isvc-60a5c-predictor.18a82b00cb0e5e52 namespace: kserve-ci-e2e-test resourceVersion: "24935" uid: fc1a221c-3361-4ad5-833e-58e7988b9a2c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-20T20:36:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-60a5c-predictor namespace: kserve-ci-e2e-test resourceVersion: "24774" uid: 57f41437-414e-44f8-83b5-72c8a9416fab kind: Event lastTimestamp: "2026-04-20T20:36:35Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:36:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:36:35Z" name: error-404-isvc-60a5c-predictor.18a82b07c86f9ce5 namespace: kserve-ci-e2e-test resourceVersion: "25501" uid: 7cc682ea-1e5a-43dd-a3bd-b67e5e8a6562 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-20T20:36:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-60a5c-predictor namespace: kserve-ci-e2e-test resourceVersion: "24774" uid: 57f41437-414e-44f8-83b5-72c8a9416fab kind: Event lastTimestamp: "2026-04-20T20:36:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:36:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:36:35Z" name: error-404-isvc-60a5c-predictor.18a82b07c8d0e956 namespace: kserve-ci-e2e-test resourceVersion: "25502" uid: 6fe89450-18be-4551-b383-d6e2bae33dc1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-20T20:35:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-60a5c namespace: kserve-ci-e2e-test resourceVersion: "24545" uid: 63c5c0e3-183d-43dc-bd64-8ff083ae8109 kind: Event lastTimestamp: "2026-04-20T20:36:08Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-20T20:35:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:36:08Z" name: error-404-isvc-60a5c.18a82afd4c1f3af8 namespace: kserve-ci-e2e-test resourceVersion: "25250" uid: e3d1c728-c998-4717-b9da-54324dedfada reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-60a5c namespace: kserve-ci-e2e-test resourceVersion: "24545" uid: 63c5c0e3-183d-43dc-bd64-8ff083ae8109 kind: Event lastTimestamp: "2026-04-20T20:35:20Z" message: 'Failed to update status for InferenceService "error-404-isvc-60a5c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-60a5c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-20T20:35:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:35:20Z" name: error-404-isvc-60a5c.18a82afd4c84c2f9 namespace: kserve-ci-e2e-test resourceVersion: "24582" uid: 264ef977-b1b9-40b3-9515-72a64bd1ed32 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-60a5c namespace: kserve-ci-e2e-test resourceVersion: "24545" uid: 63c5c0e3-183d-43dc-bd64-8ff083ae8109 kind: Event lastTimestamp: "2026-04-20T20:35:20Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-60a5c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-20T20:35:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:35:20Z" name: error-404-isvc-60a5c.18a82afd4c851223 namespace: kserve-ci-e2e-test resourceVersion: "24583" uid: 17fba682-63f4-4478-b9d1-650f7c4524c3 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:36:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-60a5c namespace: kserve-ci-e2e-test resourceVersion: "25247" uid: 63c5c0e3-183d-43dc-bd64-8ff083ae8109 kind: Event lastTimestamp: "2026-04-20T20:36:08Z" message: InferenceService [error-404-isvc-60a5c] is Ready metadata: creationTimestamp: "2026-04-20T20:36:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:36:08Z" name: error-404-isvc-60a5c.18a82b087587c6dc namespace: kserve-ci-e2e-test resourceVersion: "25248" uid: f469aca9-7288-4e87-a838-3450f5589ded reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-20T20:24:55.241750Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-63de8-predictor-544568fdd5-wfwsj namespace: kserve-ci-e2e-test resourceVersion: "19335" uid: 2e558e2f-8541-47ab-bd38-141408b5dbcb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-63de8-predictor-544568fdd5-wfwsj to ip-10-0-130-227.ec2.internal metadata: creationTimestamp: "2026-04-20T20:24:55Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-20T20:24:55Z" name: error-404-isvc-63de8-predictor-544568fdd5-wfwsj.18a82a6bbe641843 namespace: kserve-ci-e2e-test resourceVersion: "19339" uid: d9536f61-48b0-441a-8a68-fb5071f5a906 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58dbb994fc-hbcnb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:24:55Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-63de8-predictor-544568fdd5-wfwsj namespace: kserve-ci-e2e-test resourceVersion: "19345" uid: 2e558e2f-8541-47ab-bd38-141408b5dbcb kind: Event lastTimestamp: "2026-04-20T20:24:55Z" message: Add eth0 [10.133.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-20T20:24:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-20T20:24:55Z" name: error-404-isvc-63de8-predictor-544568fdd5-wfwsj.18a82a6bd77e343b namespace: kserve-ci-e2e-test resourceVersion: "19353" uid: 52d06348-6fda-4852-a04e-34e073e64beb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:24:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-63de8-predictor-544568fdd5-wfwsj namespace: kserve-ci-e2e-test resourceVersion: "19337" uid: 2e558e2f-8541-47ab-bd38-141408b5dbcb kind: Event lastTimestamp: "2026-04-20T20:24:55Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-20T20:24:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:24:55Z" name: error-404-isvc-63de8-predictor-544568fdd5-wfwsj.18a82a6bd8c00a0a namespace: kserve-ci-e2e-test resourceVersion: "19355" uid: 184f7922-cd7f-4067-8b04-8585bf4cbde6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:24:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-63de8-predictor-544568fdd5-wfwsj namespace: kserve-ci-e2e-test resourceVersion: "19337" uid: 2e558e2f-8541-47ab-bd38-141408b5dbcb kind: Event lastTimestamp: "2026-04-20T20:24:55Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-20T20:24:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:24:55Z" name: error-404-isvc-63de8-predictor-544568fdd5-wfwsj.18a82a6bdbcd7084 namespace: kserve-ci-e2e-test resourceVersion: "19356" uid: f24f3dee-b6f8-4cb4-9c16-f8056dbd8786 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:24:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-63de8-predictor-544568fdd5-wfwsj namespace: kserve-ci-e2e-test resourceVersion: "19337" uid: 2e558e2f-8541-47ab-bd38-141408b5dbcb kind: Event lastTimestamp: "2026-04-20T20:24:55Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-20T20:24:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:24:55Z" name: error-404-isvc-63de8-predictor-544568fdd5-wfwsj.18a82a6bdc3d2f46 namespace: kserve-ci-e2e-test resourceVersion: "19357" uid: f2b2ef8a-1883-4599-89a9-c3cd9e822fb6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:24:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-63de8-predictor-544568fdd5-wfwsj namespace: kserve-ci-e2e-test resourceVersion: "19337" uid: 2e558e2f-8541-47ab-bd38-141408b5dbcb kind: Event lastTimestamp: "2026-04-20T20:24:55Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-20T20:24:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:24:55Z" name: error-404-isvc-63de8-predictor-544568fdd5-wfwsj.18a82a6bdc48c25e namespace: kserve-ci-e2e-test resourceVersion: "19358" uid: 4948b719-c184-4681-ba4f-489ff8fac4e0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:24:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-63de8-predictor-544568fdd5-wfwsj namespace: kserve-ci-e2e-test resourceVersion: "19337" uid: 2e558e2f-8541-47ab-bd38-141408b5dbcb kind: Event lastTimestamp: "2026-04-20T20:24:55Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-20T20:24:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:24:55Z" name: error-404-isvc-63de8-predictor-544568fdd5-wfwsj.18a82a6bdf6f751b namespace: kserve-ci-e2e-test resourceVersion: "19360" uid: 637ea45f-5b60-445a-86b4-500d29f6f215 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:24:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-63de8-predictor-544568fdd5-wfwsj namespace: kserve-ci-e2e-test resourceVersion: "19337" uid: 2e558e2f-8541-47ab-bd38-141408b5dbcb kind: Event lastTimestamp: "2026-04-20T20:24:55Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:24:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:24:55Z" name: error-404-isvc-63de8-predictor-544568fdd5-wfwsj.18a82a6bdff2546c namespace: kserve-ci-e2e-test resourceVersion: "19361" uid: bacc8555-12a5-49f9-bafd-08a73445635b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-20T20:24:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-63de8-predictor-544568fdd5-wfwsj namespace: kserve-ci-e2e-test resourceVersion: "19337" uid: 2e558e2f-8541-47ab-bd38-141408b5dbcb kind: Event lastTimestamp: "2026-04-20T20:25:33Z" message: 'Readiness probe failed: dial tcp 10.133.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-20T20:24:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:25:33Z" name: error-404-isvc-63de8-predictor-544568fdd5-wfwsj.18a82a6c2cfa70be namespace: kserve-ci-e2e-test resourceVersion: "19889" uid: 0e1a4a7c-ae71-4c1a-ad13-dc434dc5dbc8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:26:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-63de8-predictor-544568fdd5-wfwsj namespace: kserve-ci-e2e-test resourceVersion: "19337" uid: 2e558e2f-8541-47ab-bd38-141408b5dbcb kind: Event lastTimestamp: "2026-04-20T20:26:05Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-20T20:26:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:26:05Z" name: error-404-isvc-63de8-predictor-544568fdd5-wfwsj.18a82a7c16b11903 namespace: kserve-ci-e2e-test resourceVersion: "20293" uid: 798102b1-24c0-4f8a-9af8-064e47057c67 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:26:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-63de8-predictor-544568fdd5-wfwsj namespace: kserve-ci-e2e-test resourceVersion: "19337" uid: 2e558e2f-8541-47ab-bd38-141408b5dbcb kind: Event lastTimestamp: "2026-04-20T20:26:05Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:26:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:26:05Z" name: error-404-isvc-63de8-predictor-544568fdd5-wfwsj.18a82a7c16b1758d namespace: kserve-ci-e2e-test resourceVersion: "20297" uid: 13210acc-b271-474a-b0dc-cf7c7baec975 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:26:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-63de8-predictor-544568fdd5-wfwsj namespace: kserve-ci-e2e-test resourceVersion: "19337" uid: 2e558e2f-8541-47ab-bd38-141408b5dbcb kind: Event lastTimestamp: "2026-04-20T20:26:08Z" message: 'Readiness probe failed: Get "https://10.133.0.37:8643/healthz": dial tcp 10.133.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-20T20:26:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:26:08Z" name: error-404-isvc-63de8-predictor-544568fdd5-wfwsj.18a82a7cb51fb5a7 namespace: kserve-ci-e2e-test resourceVersion: "20365" uid: 8ea21514-274c-44c7-ba3e-7c1cb56931c2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:24:55Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-63de8-predictor-544568fdd5 namespace: kserve-ci-e2e-test resourceVersion: "19328" uid: 8aee7287-d96a-4bc4-a165-1db9a4449dd7 kind: Event lastTimestamp: "2026-04-20T20:24:55Z" message: 'Created pod: error-404-isvc-63de8-predictor-544568fdd5-wfwsj' metadata: creationTimestamp: "2026-04-20T20:24:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:24:55Z" name: error-404-isvc-63de8-predictor-544568fdd5.18a82a6bbde1d2d7 namespace: kserve-ci-e2e-test resourceVersion: "19336" uid: 51e3c1d6-fb20-4b2f-8934-005261ab1b90 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:24:55Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-63de8-predictor namespace: kserve-ci-e2e-test resourceVersion: "19326" uid: 1e8db593-e187-4cf9-9839-746b0b787634 kind: Event lastTimestamp: "2026-04-20T20:24:55Z" message: Scaled up replica set error-404-isvc-63de8-predictor-544568fdd5 from 0 to 1 metadata: creationTimestamp: "2026-04-20T20:24:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:24:55Z" name: error-404-isvc-63de8-predictor.18a82a6bbc60cfcc namespace: kserve-ci-e2e-test resourceVersion: "19330" uid: 80ec353b-0545-466a-b395-bdf214df158f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-20T20:25:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-63de8-predictor namespace: kserve-ci-e2e-test resourceVersion: "19327" uid: 191e508c-a859-4a95-81d7-07f14ddb1bf7 kind: Event lastTimestamp: "2026-04-20T20:25:40Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:25:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:25:40Z" name: error-404-isvc-63de8-predictor.18a82a6f3ae1682c namespace: kserve-ci-e2e-test resourceVersion: "19948" uid: 4e5a3db6-f7db-4583-9d3f-f18cb77e365c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-20T20:25:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-63de8-predictor namespace: kserve-ci-e2e-test resourceVersion: "19327" uid: 191e508c-a859-4a95-81d7-07f14ddb1bf7 kind: Event lastTimestamp: "2026-04-20T20:25:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:25:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:25:40Z" name: error-404-isvc-63de8-predictor.18a82a6f3b7412a3 namespace: kserve-ci-e2e-test resourceVersion: "19949" uid: ea6a31fb-e9c5-476e-9d95-9ae514afca3a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-63de8-predictor namespace: kserve-ci-e2e-test resourceVersion: "19522" uid: 191e508c-a859-4a95-81d7-07f14ddb1bf7 kind: Event lastTimestamp: "2026-04-20T20:25:55Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:25:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:25:55Z" name: error-404-isvc-63de8-predictor.18a82a79b77fc810 namespace: kserve-ci-e2e-test resourceVersion: "20112" uid: 021b81bd-6d0f-4e3e-8e01-54f7aca87baf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-63de8-predictor namespace: kserve-ci-e2e-test resourceVersion: "19522" uid: 191e508c-a859-4a95-81d7-07f14ddb1bf7 kind: Event lastTimestamp: "2026-04-20T20:25:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:25:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:25:55Z" name: error-404-isvc-63de8-predictor.18a82a79b7d8ba77 namespace: kserve-ci-e2e-test resourceVersion: "20114" uid: 9f082f51-1b11-4361-a988-ba163fa49d43 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-20T20:24:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-63de8 namespace: kserve-ci-e2e-test resourceVersion: "19309" uid: a8ad59d3-8e09-4112-96dc-1fb62753ced5 kind: Event lastTimestamp: "2026-04-20T20:25:43Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-20T20:24:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:25:43Z" name: error-404-isvc-63de8.18a82a6bbc4209ee namespace: kserve-ci-e2e-test resourceVersion: "19989" uid: 8837fd88-d012-4b70-a4ec-0018b66851e6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:24:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-63de8 namespace: kserve-ci-e2e-test resourceVersion: "19309" uid: a8ad59d3-8e09-4112-96dc-1fb62753ced5 kind: Event lastTimestamp: "2026-04-20T20:24:55Z" message: 'Failed to update status for InferenceService "error-404-isvc-63de8": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-63de8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-20T20:24:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:24:55Z" name: error-404-isvc-63de8.18a82a6bbcd73a06 namespace: kserve-ci-e2e-test resourceVersion: "19332" uid: 40899b30-c160-4b94-8672-33869605b877 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:24:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-63de8 namespace: kserve-ci-e2e-test resourceVersion: "19309" uid: a8ad59d3-8e09-4112-96dc-1fb62753ced5 kind: Event lastTimestamp: "2026-04-20T20:24:55Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-63de8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-20T20:24:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:24:55Z" name: error-404-isvc-63de8.18a82a6bbcd784d0 namespace: kserve-ci-e2e-test resourceVersion: "19333" uid: 511c3906-fdb4-4e97-9d85-b2d2934e233c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-63de8 namespace: kserve-ci-e2e-test resourceVersion: "19987" uid: a8ad59d3-8e09-4112-96dc-1fb62753ced5 kind: Event lastTimestamp: "2026-04-20T20:25:43Z" message: InferenceService [error-404-isvc-63de8] is Ready metadata: creationTimestamp: "2026-04-20T20:25:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:25:43Z" name: error-404-isvc-63de8.18a82a76e79bd7d5 namespace: kserve-ci-e2e-test resourceVersion: "19988" uid: a0f480c5-42ea-40fd-b79b-8ae0f2c58d00 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-20T20:24:15.464939Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7541f-predictor-6b4d6f7547-62k8s namespace: kserve-ci-e2e-test resourceVersion: "18886" uid: 55817dee-9562-415a-aa34-340f14b7b88f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-7541f-predictor-6b4d6f7547-62k8s to ip-10-0-130-227.ec2.internal metadata: creationTimestamp: "2026-04-20T20:24:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-20T20:24:15Z" name: error-404-isvc-7541f-predictor-6b4d6f7547-62k8s.18a82a627b822bae namespace: kserve-ci-e2e-test resourceVersion: "18890" uid: e15a9b3a-bc96-4451-a720-585c4ef85dc1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58dbb994fc-hbcnb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:24:15Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7541f-predictor-6b4d6f7547-62k8s namespace: kserve-ci-e2e-test resourceVersion: "18896" uid: 55817dee-9562-415a-aa34-340f14b7b88f kind: Event lastTimestamp: "2026-04-20T20:24:15Z" message: Add eth0 [10.133.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-20T20:24:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-20T20:24:15Z" name: error-404-isvc-7541f-predictor-6b4d6f7547-62k8s.18a82a6294cb9c85 namespace: kserve-ci-e2e-test resourceVersion: "18907" uid: b3f999da-dc09-44b3-b41d-db1e60b9878c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:24:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7541f-predictor-6b4d6f7547-62k8s namespace: kserve-ci-e2e-test resourceVersion: "18888" uid: 55817dee-9562-415a-aa34-340f14b7b88f kind: Event lastTimestamp: "2026-04-20T20:24:15Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-20T20:24:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:24:15Z" name: error-404-isvc-7541f-predictor-6b4d6f7547-62k8s.18a82a6296126303 namespace: kserve-ci-e2e-test resourceVersion: "18909" uid: b265b96c-04a1-41c0-a0e9-b3151ad9cd44 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:24:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7541f-predictor-6b4d6f7547-62k8s namespace: kserve-ci-e2e-test resourceVersion: "18888" uid: 55817dee-9562-415a-aa34-340f14b7b88f kind: Event lastTimestamp: "2026-04-20T20:24:15Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-20T20:24:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:24:15Z" name: error-404-isvc-7541f-predictor-6b4d6f7547-62k8s.18a82a6298fb9ffa namespace: kserve-ci-e2e-test resourceVersion: "18910" uid: 85805bb1-630d-4854-8deb-c809609e1594 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:24:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7541f-predictor-6b4d6f7547-62k8s namespace: kserve-ci-e2e-test resourceVersion: "18888" uid: 55817dee-9562-415a-aa34-340f14b7b88f kind: Event lastTimestamp: "2026-04-20T20:24:15Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-20T20:24:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:24:15Z" name: error-404-isvc-7541f-predictor-6b4d6f7547-62k8s.18a82a629969e78d namespace: kserve-ci-e2e-test resourceVersion: "18911" uid: 2ae2507e-b03c-48c9-898c-e78c2eddfe8f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:24:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7541f-predictor-6b4d6f7547-62k8s namespace: kserve-ci-e2e-test resourceVersion: "18888" uid: 55817dee-9562-415a-aa34-340f14b7b88f kind: Event lastTimestamp: "2026-04-20T20:24:15Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-20T20:24:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:24:15Z" name: error-404-isvc-7541f-predictor-6b4d6f7547-62k8s.18a82a6299748145 namespace: kserve-ci-e2e-test resourceVersion: "18912" uid: 1d47d502-33fc-4119-8fcc-5bd590cf9d3a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:24:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7541f-predictor-6b4d6f7547-62k8s namespace: kserve-ci-e2e-test resourceVersion: "18888" uid: 55817dee-9562-415a-aa34-340f14b7b88f kind: Event lastTimestamp: "2026-04-20T20:24:16Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-20T20:24:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:24:16Z" name: error-404-isvc-7541f-predictor-6b4d6f7547-62k8s.18a82a629e2a7247 namespace: kserve-ci-e2e-test resourceVersion: "18913" uid: 341af3d6-56b7-4c82-b4c2-2ec8af45a164 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:24:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7541f-predictor-6b4d6f7547-62k8s namespace: kserve-ci-e2e-test resourceVersion: "18888" uid: 55817dee-9562-415a-aa34-340f14b7b88f kind: Event lastTimestamp: "2026-04-20T20:24:16Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:24:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:24:16Z" name: error-404-isvc-7541f-predictor-6b4d6f7547-62k8s.18a82a629e9caee8 namespace: kserve-ci-e2e-test resourceVersion: "18914" uid: aaf6f8b7-e2ba-4bb6-a8c3-2711b9b59785 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-20T20:24:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7541f-predictor-6b4d6f7547-62k8s namespace: kserve-ci-e2e-test resourceVersion: "18888" uid: 55817dee-9562-415a-aa34-340f14b7b88f kind: Event lastTimestamp: "2026-04-20T20:24:53Z" message: 'Readiness probe failed: dial tcp 10.133.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-20T20:24:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:24:53Z" name: error-404-isvc-7541f-predictor-6b4d6f7547-62k8s.18a82a63100bf0a7 namespace: kserve-ci-e2e-test resourceVersion: "19230" uid: f4360d5b-661e-4590-968a-f2271cac0d02 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7541f-predictor-6b4d6f7547-62k8s namespace: kserve-ci-e2e-test resourceVersion: "18888" uid: 55817dee-9562-415a-aa34-340f14b7b88f kind: Event lastTimestamp: "2026-04-20T20:25:25Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-20T20:25:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:25:25Z" name: error-404-isvc-7541f-predictor-6b4d6f7547-62k8s.18a82a72d6a662ee namespace: kserve-ci-e2e-test resourceVersion: "19777" uid: 2c8437ca-c754-4ad2-8993-6d8bfcca9ef9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7541f-predictor-6b4d6f7547-62k8s namespace: kserve-ci-e2e-test resourceVersion: "18888" uid: 55817dee-9562-415a-aa34-340f14b7b88f kind: Event lastTimestamp: "2026-04-20T20:25:25Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:25:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:25:25Z" name: error-404-isvc-7541f-predictor-6b4d6f7547-62k8s.18a82a72d6a6de52 namespace: kserve-ci-e2e-test resourceVersion: "19778" uid: 39a40584-3ebb-4d38-9fc3-f24c94ce132c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7541f-predictor-6b4d6f7547-62k8s namespace: kserve-ci-e2e-test resourceVersion: "18888" uid: 55817dee-9562-415a-aa34-340f14b7b88f kind: Event lastTimestamp: "2026-04-20T20:25:28Z" message: 'Readiness probe failed: Get "https://10.133.0.36:8643/healthz": dial tcp 10.133.0.36:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-20T20:25:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:25:28Z" name: error-404-isvc-7541f-predictor-6b4d6f7547-62k8s.18a82a739835e40e namespace: kserve-ci-e2e-test resourceVersion: "19854" uid: 51656f7e-1388-4028-9f84-3ff722a8ccc4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:24:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-7541f-predictor-6b4d6f7547 namespace: kserve-ci-e2e-test resourceVersion: "18879" uid: 3ed40cea-e2e0-40af-8777-accd88e0971c kind: Event lastTimestamp: "2026-04-20T20:24:15Z" message: 'Created pod: error-404-isvc-7541f-predictor-6b4d6f7547-62k8s' metadata: creationTimestamp: "2026-04-20T20:24:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:24:15Z" name: error-404-isvc-7541f-predictor-6b4d6f7547.18a82a627b174e15 namespace: kserve-ci-e2e-test resourceVersion: "18887" uid: 620d841c-0bc5-4748-8a51-5404d53569a4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:24:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-7541f-predictor namespace: kserve-ci-e2e-test resourceVersion: "18876" uid: cd231d56-651f-4d60-89a2-8bd5c74fe568 kind: Event lastTimestamp: "2026-04-20T20:24:15Z" message: Scaled up replica set error-404-isvc-7541f-predictor-6b4d6f7547 from 0 to 1 metadata: creationTimestamp: "2026-04-20T20:24:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:24:15Z" name: error-404-isvc-7541f-predictor.18a82a627986f3bf namespace: kserve-ci-e2e-test resourceVersion: "18881" uid: a321b859-7ed6-4ab9-8c88-424c5682bf61 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:24:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7541f-predictor namespace: kserve-ci-e2e-test resourceVersion: "18878" uid: a65d6566-1868-41a6-89b1-93c6e0a87946 kind: Event lastTimestamp: "2026-04-20T20:24:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:24:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:24:45Z" name: error-404-isvc-7541f-predictor.18a82a65f7fc6f2d namespace: kserve-ci-e2e-test resourceVersion: "19167" uid: 9d71e977-da1f-4cff-860d-bdddccc459cd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:24:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7541f-predictor namespace: kserve-ci-e2e-test resourceVersion: "18878" uid: a65d6566-1868-41a6-89b1-93c6e0a87946 kind: Event lastTimestamp: "2026-04-20T20:24:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:24:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:24:45Z" name: error-404-isvc-7541f-predictor.18a82a65f87778b7 namespace: kserve-ci-e2e-test resourceVersion: "19168" uid: 5b490b63-a220-4624-9435-79e0ac93e0de reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:25:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7541f-predictor namespace: kserve-ci-e2e-test resourceVersion: "19057" uid: a65d6566-1868-41a6-89b1-93c6e0a87946 kind: Event lastTimestamp: "2026-04-20T20:25:15Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:25:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:25:15Z" name: error-404-isvc-7541f-predictor.18a82a6cf5ead286 namespace: kserve-ci-e2e-test resourceVersion: "19598" uid: ed392678-334b-431f-93f2-1c04e2c1f527 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:25:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7541f-predictor namespace: kserve-ci-e2e-test resourceVersion: "19057" uid: a65d6566-1868-41a6-89b1-93c6e0a87946 kind: Event lastTimestamp: "2026-04-20T20:25:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:25:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:25:15Z" name: error-404-isvc-7541f-predictor.18a82a6cf6830d74 namespace: kserve-ci-e2e-test resourceVersion: "19599" uid: 7265f1f7-a10e-42b7-ac4b-4d316903f29f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-20T20:24:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7541f namespace: kserve-ci-e2e-test resourceVersion: "18860" uid: b36f5760-6ac5-438a-8887-7804b604d225 kind: Event lastTimestamp: "2026-04-20T20:25:04Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-20T20:24:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:25:04Z" name: error-404-isvc-7541f.18a82a6279613f99 namespace: kserve-ci-e2e-test resourceVersion: "19468" uid: 05d4ef15-eac1-48b4-b750-6dca7d6a7cae reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:24:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7541f namespace: kserve-ci-e2e-test resourceVersion: "18860" uid: b36f5760-6ac5-438a-8887-7804b604d225 kind: Event lastTimestamp: "2026-04-20T20:24:15Z" message: 'Failed to update status for InferenceService "error-404-isvc-7541f": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-7541f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-20T20:24:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:24:15Z" name: error-404-isvc-7541f.18a82a6279c2817a namespace: kserve-ci-e2e-test resourceVersion: "18883" uid: 67d4b4c4-b720-4f98-8f7f-75fb9d8a965e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:24:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7541f namespace: kserve-ci-e2e-test resourceVersion: "18860" uid: b36f5760-6ac5-438a-8887-7804b604d225 kind: Event lastTimestamp: "2026-04-20T20:24:15Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-7541f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-20T20:24:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:24:15Z" name: error-404-isvc-7541f.18a82a6279c2d590 namespace: kserve-ci-e2e-test resourceVersion: "18884" uid: 542e88dd-2715-4cf7-9092-1a241b474d8f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7541f namespace: kserve-ci-e2e-test resourceVersion: "19465" uid: b36f5760-6ac5-438a-8887-7804b604d225 kind: Event lastTimestamp: "2026-04-20T20:25:04Z" message: InferenceService [error-404-isvc-7541f] is Ready metadata: creationTimestamp: "2026-04-20T20:25:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:25:04Z" name: error-404-isvc-7541f.18a82a6dcaae0d99 namespace: kserve-ci-e2e-test resourceVersion: "19466" uid: a1d45f28-60db-4f1e-b096-7167deebb74f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-20T20:15:00.517760Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-9290e-predictor-df657ff87-2cvh9 namespace: kserve-ci-e2e-test resourceVersion: "14481" uid: e3e06a0f-33be-4b0a-995f-5b9df5292665 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-9290e-predictor-df657ff87-2cvh9 to ip-10-0-130-227.ec2.internal metadata: creationTimestamp: "2026-04-20T20:15:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-20T20:15:00Z" name: error-404-isvc-9290e-predictor-df657ff87-2cvh9.18a829e146143180 namespace: kserve-ci-e2e-test resourceVersion: "14485" uid: 9c75fc8b-2dcc-4cb9-af62-f6afa4729960 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58dbb994fc-hbcnb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:00Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-9290e-predictor-df657ff87-2cvh9 namespace: kserve-ci-e2e-test resourceVersion: "14491" uid: e3e06a0f-33be-4b0a-995f-5b9df5292665 kind: Event lastTimestamp: "2026-04-20T20:15:00Z" message: Add eth0 [10.133.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-20T20:15:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-20T20:15:00Z" name: error-404-isvc-9290e-predictor-df657ff87-2cvh9.18a829e15f37ee85 namespace: kserve-ci-e2e-test resourceVersion: "14502" uid: c872d74a-7fda-409a-87db-0c39645d5f5f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9290e-predictor-df657ff87-2cvh9 namespace: kserve-ci-e2e-test resourceVersion: "14483" uid: e3e06a0f-33be-4b0a-995f-5b9df5292665 kind: Event lastTimestamp: "2026-04-20T20:15:00Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-20T20:15:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:15:00Z" name: error-404-isvc-9290e-predictor-df657ff87-2cvh9.18a829e16085f357 namespace: kserve-ci-e2e-test resourceVersion: "14504" uid: deaef917-41d3-4766-a4b5-b5047d17b05a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9290e-predictor-df657ff87-2cvh9 namespace: kserve-ci-e2e-test resourceVersion: "14483" uid: e3e06a0f-33be-4b0a-995f-5b9df5292665 kind: Event lastTimestamp: "2026-04-20T20:15:01Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-20T20:15:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:15:01Z" name: error-404-isvc-9290e-predictor-df657ff87-2cvh9.18a829e16336c717 namespace: kserve-ci-e2e-test resourceVersion: "14505" uid: 30a487e4-61ba-4112-87fb-00f51f9a83c2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9290e-predictor-df657ff87-2cvh9 namespace: kserve-ci-e2e-test resourceVersion: "14483" uid: e3e06a0f-33be-4b0a-995f-5b9df5292665 kind: Event lastTimestamp: "2026-04-20T20:15:01Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-20T20:15:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:15:01Z" name: error-404-isvc-9290e-predictor-df657ff87-2cvh9.18a829e163a2d7bd namespace: kserve-ci-e2e-test resourceVersion: "14506" uid: 18de5fac-f51e-4f4b-b676-f8efeb345d2e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9290e-predictor-df657ff87-2cvh9 namespace: kserve-ci-e2e-test resourceVersion: "14483" uid: e3e06a0f-33be-4b0a-995f-5b9df5292665 kind: Event lastTimestamp: "2026-04-20T20:15:01Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-20T20:15:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:15:01Z" name: error-404-isvc-9290e-predictor-df657ff87-2cvh9.18a829e163ae8f32 namespace: kserve-ci-e2e-test resourceVersion: "14507" uid: a4890c1c-1af7-4d77-a4d9-008da732001e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9290e-predictor-df657ff87-2cvh9 namespace: kserve-ci-e2e-test resourceVersion: "14483" uid: e3e06a0f-33be-4b0a-995f-5b9df5292665 kind: Event lastTimestamp: "2026-04-20T20:15:01Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-20T20:15:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:15:01Z" name: error-404-isvc-9290e-predictor-df657ff87-2cvh9.18a829e1664a98d8 namespace: kserve-ci-e2e-test resourceVersion: "14509" uid: 7800db59-c5cd-4cec-94e1-9df4994407ec reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9290e-predictor-df657ff87-2cvh9 namespace: kserve-ci-e2e-test resourceVersion: "14483" uid: e3e06a0f-33be-4b0a-995f-5b9df5292665 kind: Event lastTimestamp: "2026-04-20T20:15:01Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:15:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:15:01Z" name: error-404-isvc-9290e-predictor-df657ff87-2cvh9.18a829e166bcccc6 namespace: kserve-ci-e2e-test resourceVersion: "14510" uid: 3409318f-21e0-4552-95f3-9f93269a0bc2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-20T20:15:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9290e-predictor-df657ff87-2cvh9 namespace: kserve-ci-e2e-test resourceVersion: "14483" uid: e3e06a0f-33be-4b0a-995f-5b9df5292665 kind: Event lastTimestamp: "2026-04-20T20:24:18Z" message: 'Readiness probe failed: dial tcp 10.133.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-20T20:15:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:24:18Z" name: error-404-isvc-9290e-predictor-df657ff87-2cvh9.18a829e1a2b9f221 namespace: kserve-ci-e2e-test resourceVersion: "18949" uid: 29c8e880-ca7a-4a51-9002-177eae7c0c50 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:24:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9290e-predictor-df657ff87-2cvh9 namespace: kserve-ci-e2e-test resourceVersion: "14483" uid: e3e06a0f-33be-4b0a-995f-5b9df5292665 kind: Event lastTimestamp: "2026-04-20T20:24:15Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-20T20:24:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:24:15Z" name: error-404-isvc-9290e-predictor-df657ff87-2cvh9.18a82a6274aade57 namespace: kserve-ci-e2e-test resourceVersion: "18858" uid: 672e7f0b-80f2-49e2-bd75-7c7908780996 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:24:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9290e-predictor-df657ff87-2cvh9 namespace: kserve-ci-e2e-test resourceVersion: "14483" uid: e3e06a0f-33be-4b0a-995f-5b9df5292665 kind: Event lastTimestamp: "2026-04-20T20:24:15Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:24:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:24:15Z" name: error-404-isvc-9290e-predictor-df657ff87-2cvh9.18a82a6274ab6fe3 namespace: kserve-ci-e2e-test resourceVersion: "18859" uid: dbc0c019-ce07-4cef-82ed-18dfe4b77498 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:24:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9290e-predictor-df657ff87-2cvh9 namespace: kserve-ci-e2e-test resourceVersion: "14483" uid: e3e06a0f-33be-4b0a-995f-5b9df5292665 kind: Event lastTimestamp: "2026-04-20T20:24:18Z" message: 'Readiness probe failed: Get "https://10.133.0.34:8643/healthz": dial tcp 10.133.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-20T20:24:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:24:18Z" name: error-404-isvc-9290e-predictor-df657ff87-2cvh9.18a82a63173f86ff namespace: kserve-ci-e2e-test resourceVersion: "18948" uid: 316558ab-48b1-4229-9bf4-08d5ded099ba reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-9290e-predictor-df657ff87 namespace: kserve-ci-e2e-test resourceVersion: "14473" uid: 65c603ef-b4df-4fa2-8653-f8b6f198ff3e kind: Event lastTimestamp: "2026-04-20T20:15:00Z" message: 'Created pod: error-404-isvc-9290e-predictor-df657ff87-2cvh9' metadata: creationTimestamp: "2026-04-20T20:15:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:15:00Z" name: error-404-isvc-9290e-predictor-df657ff87.18a829e145768d42 namespace: kserve-ci-e2e-test resourceVersion: "14482" uid: fce2f02f-fc5a-4ce6-b1aa-59a8b17e1716 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-9290e-predictor namespace: kserve-ci-e2e-test resourceVersion: "14472" uid: 5dcd3f01-cd85-4a80-a6c5-40a006892159 kind: Event lastTimestamp: "2026-04-20T20:15:00Z" message: Scaled up replica set error-404-isvc-9290e-predictor-df657ff87 from 0 to 1 metadata: creationTimestamp: "2026-04-20T20:15:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:15:00Z" name: error-404-isvc-9290e-predictor.18a829e1425ba5e7 namespace: kserve-ci-e2e-test resourceVersion: "14475" uid: fbcdc250-d4ce-4bf2-9b36-2f18b12bdcd9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:15:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9290e-predictor namespace: kserve-ci-e2e-test resourceVersion: "14474" uid: d480d5a7-3376-4225-9224-427091394dc5 kind: Event lastTimestamp: "2026-04-20T20:15:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:15:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:15:30Z" name: error-404-isvc-9290e-predictor.18a829e4c1a3bbd6 namespace: kserve-ci-e2e-test resourceVersion: "14830" uid: 009039c3-69b8-4258-bce9-3cd281c0a6c2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:15:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9290e-predictor namespace: kserve-ci-e2e-test resourceVersion: "14474" uid: d480d5a7-3376-4225-9224-427091394dc5 kind: Event lastTimestamp: "2026-04-20T20:15:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:15:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:15:30Z" name: error-404-isvc-9290e-predictor.18a829e4c20fe85e namespace: kserve-ci-e2e-test resourceVersion: "14831" uid: 0f963d7c-94e5-427c-8039-1dd7d6669a63 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-20T20:15:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9290e-predictor namespace: kserve-ci-e2e-test resourceVersion: "14676" uid: d480d5a7-3376-4225-9224-427091394dc5 kind: Event lastTimestamp: "2026-04-20T20:16:15Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:15:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:16:15Z" name: error-404-isvc-9290e-predictor.18a829ebbf86b11b namespace: kserve-ci-e2e-test resourceVersion: "15433" uid: c31b23d5-5067-476b-a719-f04bc828229b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-20T20:15:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9290e-predictor namespace: kserve-ci-e2e-test resourceVersion: "14676" uid: d480d5a7-3376-4225-9224-427091394dc5 kind: Event lastTimestamp: "2026-04-20T20:16:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:15:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:16:15Z" name: error-404-isvc-9290e-predictor.18a829ebbfedf7be namespace: kserve-ci-e2e-test resourceVersion: "15434" uid: 23a22855-f4e0-46bf-adfe-0d57924de229 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-20T20:15:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9290e namespace: kserve-ci-e2e-test resourceVersion: "14456" uid: 1270f4b1-d0fa-4194-85f1-697ccf95977c kind: Event lastTimestamp: "2026-04-20T20:15:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-20T20:15:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:15:48Z" name: error-404-isvc-9290e.18a829e1430ec6f8 namespace: kserve-ci-e2e-test resourceVersion: "15160" uid: 54f7bda7-fa00-481f-bc31-43fa9849def4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9290e namespace: kserve-ci-e2e-test resourceVersion: "15157" uid: 1270f4b1-d0fa-4194-85f1-697ccf95977c kind: Event lastTimestamp: "2026-04-20T20:15:48Z" message: InferenceService [error-404-isvc-9290e] is Ready metadata: creationTimestamp: "2026-04-20T20:15:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:15:48Z" name: error-404-isvc-9290e.18a829ec5d4d4fe3 namespace: kserve-ci-e2e-test resourceVersion: "15158" uid: fd483d0a-b1dc-44b4-b150-8d7eb0a76c1d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-20T20:34:40.601926Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ab6b8-predictor-7fb5d66599-zms7g namespace: kserve-ci-e2e-test resourceVersion: "24137" uid: bf9f6842-25d2-484c-8a66-9010dadf1cb9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-ab6b8-predictor-7fb5d66599-zms7g to ip-10-0-130-227.ec2.internal metadata: creationTimestamp: "2026-04-20T20:34:40Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-20T20:34:40Z" name: error-404-isvc-ab6b8-predictor-7fb5d66599-zms7g.18a82af408938bfb namespace: kserve-ci-e2e-test resourceVersion: "24141" uid: bb076467-e6a0-40b6-8256-c27c96248de2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58dbb994fc-hbcnb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:34:40Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ab6b8-predictor-7fb5d66599-zms7g namespace: kserve-ci-e2e-test resourceVersion: "24139" uid: bf9f6842-25d2-484c-8a66-9010dadf1cb9 kind: Event lastTimestamp: "2026-04-20T20:34:40Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-ab6b8-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-20T20:34:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:34:40Z" name: error-404-isvc-ab6b8-predictor-7fb5d66599-zms7g.18a82af41528532f namespace: kserve-ci-e2e-test resourceVersion: "24151" uid: fdbd082d-534a-48d6-9bc7-510cce986700 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:34:41Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ab6b8-predictor-7fb5d66599-zms7g namespace: kserve-ci-e2e-test resourceVersion: "24146" uid: bf9f6842-25d2-484c-8a66-9010dadf1cb9 kind: Event lastTimestamp: "2026-04-20T20:34:41Z" message: Add eth0 [10.133.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-20T20:34:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-20T20:34:41Z" name: error-404-isvc-ab6b8-predictor-7fb5d66599-zms7g.18a82af445b8bdf3 namespace: kserve-ci-e2e-test resourceVersion: "24170" uid: 857c7df3-7033-4e49-9659-30b999a308e7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:34:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ab6b8-predictor-7fb5d66599-zms7g namespace: kserve-ci-e2e-test resourceVersion: "24139" uid: bf9f6842-25d2-484c-8a66-9010dadf1cb9 kind: Event lastTimestamp: "2026-04-20T20:34:41Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-20T20:34:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:34:41Z" name: error-404-isvc-ab6b8-predictor-7fb5d66599-zms7g.18a82af446fc7e22 namespace: kserve-ci-e2e-test resourceVersion: "24173" uid: a1e596be-3eb8-467b-ba27-6fdbba59aeaa reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:34:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ab6b8-predictor-7fb5d66599-zms7g namespace: kserve-ci-e2e-test resourceVersion: "24139" uid: bf9f6842-25d2-484c-8a66-9010dadf1cb9 kind: Event lastTimestamp: "2026-04-20T20:34:41Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-20T20:34:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:34:41Z" name: error-404-isvc-ab6b8-predictor-7fb5d66599-zms7g.18a82af44ae62a23 namespace: kserve-ci-e2e-test resourceVersion: "24175" uid: c0dee64a-7ae3-408e-a47d-66573f3ece36 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:34:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ab6b8-predictor-7fb5d66599-zms7g namespace: kserve-ci-e2e-test resourceVersion: "24139" uid: bf9f6842-25d2-484c-8a66-9010dadf1cb9 kind: Event lastTimestamp: "2026-04-20T20:34:41Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-20T20:34:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:34:41Z" name: error-404-isvc-ab6b8-predictor-7fb5d66599-zms7g.18a82af44b7e8d9a namespace: kserve-ci-e2e-test resourceVersion: "24176" uid: 8001b681-8593-489c-b264-5e32a9ec1e4a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:34:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ab6b8-predictor-7fb5d66599-zms7g namespace: kserve-ci-e2e-test resourceVersion: "24139" uid: bf9f6842-25d2-484c-8a66-9010dadf1cb9 kind: Event lastTimestamp: "2026-04-20T20:34:41Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-20T20:34:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:34:41Z" name: error-404-isvc-ab6b8-predictor-7fb5d66599-zms7g.18a82af44b8ae67a namespace: kserve-ci-e2e-test resourceVersion: "24177" uid: 5853901a-d072-48d7-8a73-1d49b8844d11 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:34:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ab6b8-predictor-7fb5d66599-zms7g namespace: kserve-ci-e2e-test resourceVersion: "24139" uid: bf9f6842-25d2-484c-8a66-9010dadf1cb9 kind: Event lastTimestamp: "2026-04-20T20:34:41Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-20T20:34:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:34:41Z" name: error-404-isvc-ab6b8-predictor-7fb5d66599-zms7g.18a82af44ee8209f namespace: kserve-ci-e2e-test resourceVersion: "24179" uid: d7578271-4d91-456c-b7b7-03a1d1df752d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:34:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ab6b8-predictor-7fb5d66599-zms7g namespace: kserve-ci-e2e-test resourceVersion: "24139" uid: bf9f6842-25d2-484c-8a66-9010dadf1cb9 kind: Event lastTimestamp: "2026-04-20T20:34:41Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:34:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:34:41Z" name: error-404-isvc-ab6b8-predictor-7fb5d66599-zms7g.18a82af44f55bd9b namespace: kserve-ci-e2e-test resourceVersion: "24180" uid: 4e1c41dc-b7af-4b31-91eb-4672e9e8656f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-20T20:34:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ab6b8-predictor-7fb5d66599-zms7g namespace: kserve-ci-e2e-test resourceVersion: "24139" uid: bf9f6842-25d2-484c-8a66-9010dadf1cb9 kind: Event lastTimestamp: "2026-04-20T20:35:19Z" message: 'Readiness probe failed: dial tcp 10.133.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-20T20:34:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:35:19Z" name: error-404-isvc-ab6b8-predictor-7fb5d66599-zms7g.18a82af49dfa29ec namespace: kserve-ci-e2e-test resourceVersion: "24484" uid: 7380399d-3f3a-43cb-99d1-7ef470c26442 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ab6b8-predictor-7fb5d66599-zms7g namespace: kserve-ci-e2e-test resourceVersion: "24139" uid: bf9f6842-25d2-484c-8a66-9010dadf1cb9 kind: Event lastTimestamp: "2026-04-20T20:35:50Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-20T20:35:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:35:50Z" name: error-404-isvc-ab6b8-predictor-7fb5d66599-zms7g.18a82b045d791be5 namespace: kserve-ci-e2e-test resourceVersion: "24973" uid: 55ad7d50-cd27-4323-8f70-7d27b2bf6145 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ab6b8-predictor-7fb5d66599-zms7g namespace: kserve-ci-e2e-test resourceVersion: "24139" uid: bf9f6842-25d2-484c-8a66-9010dadf1cb9 kind: Event lastTimestamp: "2026-04-20T20:35:50Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:35:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:35:50Z" name: error-404-isvc-ab6b8-predictor-7fb5d66599-zms7g.18a82b045d79939e namespace: kserve-ci-e2e-test resourceVersion: "24976" uid: 00a3ff45-f0f1-4e10-9591-3a3d20197e98 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ab6b8-predictor-7fb5d66599-zms7g namespace: kserve-ci-e2e-test resourceVersion: "24139" uid: bf9f6842-25d2-484c-8a66-9010dadf1cb9 kind: Event lastTimestamp: "2026-04-20T20:35:54Z" message: 'Readiness probe failed: Get "https://10.133.0.40:8643/healthz": dial tcp 10.133.0.40:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-20T20:35:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:35:54Z" name: error-404-isvc-ab6b8-predictor-7fb5d66599-zms7g.18a82b0526388160 namespace: kserve-ci-e2e-test resourceVersion: "25104" uid: f2648fea-6356-4e0c-8658-d186db02e3a5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:34:40Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-ab6b8-predictor-7fb5d66599 namespace: kserve-ci-e2e-test resourceVersion: "24129" uid: 8ec21858-50ee-4019-8463-b9f5c7d27bfb kind: Event lastTimestamp: "2026-04-20T20:34:40Z" message: 'Created pod: error-404-isvc-ab6b8-predictor-7fb5d66599-zms7g' metadata: creationTimestamp: "2026-04-20T20:34:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:34:40Z" name: error-404-isvc-ab6b8-predictor-7fb5d66599.18a82af4082b8921 namespace: kserve-ci-e2e-test resourceVersion: "24138" uid: a4c924d9-9a23-4d65-bd9c-03e62910bbe3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:34:40Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-ab6b8-predictor namespace: kserve-ci-e2e-test resourceVersion: "24124" uid: 1a2c1ebe-e36e-45b6-8e33-ba671a447bcb kind: Event lastTimestamp: "2026-04-20T20:34:40Z" message: Scaled up replica set error-404-isvc-ab6b8-predictor-7fb5d66599 from 0 to 1 metadata: creationTimestamp: "2026-04-20T20:34:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:34:40Z" name: error-404-isvc-ab6b8-predictor.18a82af4066bbe13 namespace: kserve-ci-e2e-test resourceVersion: "24130" uid: c10a9d12-ce9e-449d-ba4a-fdd95c6fb454 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:34:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ab6b8-predictor namespace: kserve-ci-e2e-test resourceVersion: "24128" uid: 8311a491-f0d6-4300-8fae-fa67d44d6b74 kind: Event lastTimestamp: "2026-04-20T20:35:10Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:34:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:35:10Z" name: error-404-isvc-ab6b8-predictor.18a82af784fc9039 namespace: kserve-ci-e2e-test resourceVersion: "24420" uid: f32bcb0a-b27b-4c12-b1cb-1dca983da02a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:34:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ab6b8-predictor namespace: kserve-ci-e2e-test resourceVersion: "24128" uid: 8311a491-f0d6-4300-8fae-fa67d44d6b74 kind: Event lastTimestamp: "2026-04-20T20:35:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:34:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:35:10Z" name: error-404-isvc-ab6b8-predictor.18a82af7855c7a95 namespace: kserve-ci-e2e-test resourceVersion: "24421" uid: 8a4dfe86-6db5-401d-a1ab-89bc9025959c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:35:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ab6b8-predictor namespace: kserve-ci-e2e-test resourceVersion: "24308" uid: 8311a491-f0d6-4300-8fae-fa67d44d6b74 kind: Event lastTimestamp: "2026-04-20T20:35:40Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:35:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:35:40Z" name: error-404-isvc-ab6b8-predictor.18a82afe82ac136f namespace: kserve-ci-e2e-test resourceVersion: "24840" uid: feb42edb-c340-4a67-b910-5597ed371f98 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:35:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ab6b8-predictor namespace: kserve-ci-e2e-test resourceVersion: "24308" uid: 8311a491-f0d6-4300-8fae-fa67d44d6b74 kind: Event lastTimestamp: "2026-04-20T20:35:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:35:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:35:40Z" name: error-404-isvc-ab6b8-predictor.18a82afe830fb81a namespace: kserve-ci-e2e-test resourceVersion: "24841" uid: 8579b718-35c5-44f9-a647-aa9a01eb740a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-20T20:34:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ab6b8 namespace: kserve-ci-e2e-test resourceVersion: "24099" uid: 26a8a67e-e67e-45e3-97a2-010d284dce86 kind: Event lastTimestamp: "2026-04-20T20:35:29Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-20T20:34:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:35:29Z" name: error-404-isvc-ab6b8.18a82af4066cd92b namespace: kserve-ci-e2e-test resourceVersion: "24721" uid: 72081fa9-7ddb-435f-a616-f5c99780cced reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:34:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ab6b8 namespace: kserve-ci-e2e-test resourceVersion: "24099" uid: 26a8a67e-e67e-45e3-97a2-010d284dce86 kind: Event lastTimestamp: "2026-04-20T20:34:40Z" message: 'Failed to update status for InferenceService "error-404-isvc-ab6b8": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-ab6b8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-20T20:34:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:34:40Z" name: error-404-isvc-ab6b8.18a82af406fdfa72 namespace: kserve-ci-e2e-test resourceVersion: "24134" uid: afd3e9c3-031a-4dc6-904c-31189168171a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:34:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ab6b8 namespace: kserve-ci-e2e-test resourceVersion: "24099" uid: 26a8a67e-e67e-45e3-97a2-010d284dce86 kind: Event lastTimestamp: "2026-04-20T20:34:40Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-ab6b8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-20T20:34:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:34:40Z" name: error-404-isvc-ab6b8.18a82af406fe5959 namespace: kserve-ci-e2e-test resourceVersion: "24135" uid: dea84526-e38f-42ca-9496-657818c8cb69 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ab6b8 namespace: kserve-ci-e2e-test resourceVersion: "24719" uid: 26a8a67e-e67e-45e3-97a2-010d284dce86 kind: Event lastTimestamp: "2026-04-20T20:35:29Z" message: InferenceService [error-404-isvc-ab6b8] is Ready metadata: creationTimestamp: "2026-04-20T20:35:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:35:29Z" name: error-404-isvc-ab6b8.18a82aff596662fc namespace: kserve-ci-e2e-test resourceVersion: "24720" uid: b4e75964-d18b-4d2b-bed7-933d1de15a73 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-20T20:15:40.587249Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e6f34-predictor-595bdccbdf-lf8lw namespace: kserve-ci-e2e-test resourceVersion: "15030" uid: 171726d5-6680-4524-b824-23696042dc39 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-e6f34-predictor-595bdccbdf-lf8lw to ip-10-0-130-227.ec2.internal metadata: creationTimestamp: "2026-04-20T20:15:40Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-20T20:15:40Z" name: error-404-isvc-e6f34-predictor-595bdccbdf-lf8lw.18a829ea9a681841 namespace: kserve-ci-e2e-test resourceVersion: "15038" uid: 52b3027a-61f0-4c55-ac33-00bb62c45e69 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58dbb994fc-hbcnb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:41Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e6f34-predictor-595bdccbdf-lf8lw namespace: kserve-ci-e2e-test resourceVersion: "15046" uid: 171726d5-6680-4524-b824-23696042dc39 kind: Event lastTimestamp: "2026-04-20T20:15:41Z" message: Add eth0 [10.133.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-20T20:15:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-20T20:15:41Z" name: error-404-isvc-e6f34-predictor-595bdccbdf-lf8lw.18a829eab441d056 namespace: kserve-ci-e2e-test resourceVersion: "15062" uid: fdd995d5-a264-4055-8fcb-ddf9aa4c6907 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e6f34-predictor-595bdccbdf-lf8lw namespace: kserve-ci-e2e-test resourceVersion: "15033" uid: 171726d5-6680-4524-b824-23696042dc39 kind: Event lastTimestamp: "2026-04-20T20:15:41Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-20T20:15:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:15:41Z" name: error-404-isvc-e6f34-predictor-595bdccbdf-lf8lw.18a829eab608643e namespace: kserve-ci-e2e-test resourceVersion: "15066" uid: b847aea6-6dde-42f9-b506-f8acf8d368d8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e6f34-predictor-595bdccbdf-lf8lw namespace: kserve-ci-e2e-test resourceVersion: "15033" uid: 171726d5-6680-4524-b824-23696042dc39 kind: Event lastTimestamp: "2026-04-20T20:15:41Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-20T20:15:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:15:41Z" name: error-404-isvc-e6f34-predictor-595bdccbdf-lf8lw.18a829eab9bdcc3c namespace: kserve-ci-e2e-test resourceVersion: "15067" uid: 5dd6ed32-cea1-4767-8722-6077bc6c8f00 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e6f34-predictor-595bdccbdf-lf8lw namespace: kserve-ci-e2e-test resourceVersion: "15033" uid: 171726d5-6680-4524-b824-23696042dc39 kind: Event lastTimestamp: "2026-04-20T20:15:41Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-20T20:15:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:15:41Z" name: error-404-isvc-e6f34-predictor-595bdccbdf-lf8lw.18a829eaba339e86 namespace: kserve-ci-e2e-test resourceVersion: "15068" uid: 9aed2a96-1752-47bc-982a-a5b45548d33d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e6f34-predictor-595bdccbdf-lf8lw namespace: kserve-ci-e2e-test resourceVersion: "15033" uid: 171726d5-6680-4524-b824-23696042dc39 kind: Event lastTimestamp: "2026-04-20T20:15:41Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-20T20:15:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:15:41Z" name: error-404-isvc-e6f34-predictor-595bdccbdf-lf8lw.18a829eaba4095a7 namespace: kserve-ci-e2e-test resourceVersion: "15069" uid: 2cf621df-81b3-4523-b779-45ead20dd32c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e6f34-predictor-595bdccbdf-lf8lw namespace: kserve-ci-e2e-test resourceVersion: "15033" uid: 171726d5-6680-4524-b824-23696042dc39 kind: Event lastTimestamp: "2026-04-20T20:15:41Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-20T20:15:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:15:41Z" name: error-404-isvc-e6f34-predictor-595bdccbdf-lf8lw.18a829eabd3adea9 namespace: kserve-ci-e2e-test resourceVersion: "15070" uid: 963730c1-6083-4a29-9771-88c4567c3ada reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e6f34-predictor-595bdccbdf-lf8lw namespace: kserve-ci-e2e-test resourceVersion: "15033" uid: 171726d5-6680-4524-b824-23696042dc39 kind: Event lastTimestamp: "2026-04-20T20:15:41Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:15:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:15:41Z" name: error-404-isvc-e6f34-predictor-595bdccbdf-lf8lw.18a829eabdb0a0da namespace: kserve-ci-e2e-test resourceVersion: "15071" uid: b03546cf-c8f1-4a26-b7c3-bc119157f55c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-20T20:15:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e6f34-predictor-595bdccbdf-lf8lw namespace: kserve-ci-e2e-test resourceVersion: "15033" uid: 171726d5-6680-4524-b824-23696042dc39 kind: Event lastTimestamp: "2026-04-20T20:24:58Z" message: 'Readiness probe failed: dial tcp 10.133.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-20T20:15:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:24:58Z" name: error-404-isvc-e6f34-predictor-595bdccbdf-lf8lw.18a829eafae10ef4 namespace: kserve-ci-e2e-test resourceVersion: "19402" uid: de523773-0b23-409b-adb0-a591b907d326 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:24:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e6f34-predictor-595bdccbdf-lf8lw namespace: kserve-ci-e2e-test resourceVersion: "15033" uid: 171726d5-6680-4524-b824-23696042dc39 kind: Event lastTimestamp: "2026-04-20T20:24:55Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-20T20:24:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:24:55Z" name: error-404-isvc-e6f34-predictor-595bdccbdf-lf8lw.18a82a6bb43de4e5 namespace: kserve-ci-e2e-test resourceVersion: "19275" uid: 2774fcdc-5557-42f4-a671-871ce44911de reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:24:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e6f34-predictor-595bdccbdf-lf8lw namespace: kserve-ci-e2e-test resourceVersion: "15033" uid: 171726d5-6680-4524-b824-23696042dc39 kind: Event lastTimestamp: "2026-04-20T20:24:55Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:24:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:24:55Z" name: error-404-isvc-e6f34-predictor-595bdccbdf-lf8lw.18a82a6bb43e6c64 namespace: kserve-ci-e2e-test resourceVersion: "19276" uid: 401c7aeb-067c-4c10-8385-63bab2176314 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:24:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e6f34-predictor-595bdccbdf-lf8lw namespace: kserve-ci-e2e-test resourceVersion: "15033" uid: 171726d5-6680-4524-b824-23696042dc39 kind: Event lastTimestamp: "2026-04-20T20:24:58Z" message: 'Readiness probe failed: Get "https://10.133.0.35:8643/healthz": dial tcp 10.133.0.35:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-20T20:24:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:24:58Z" name: error-404-isvc-e6f34-predictor-595bdccbdf-lf8lw.18a82a6c6f327328 namespace: kserve-ci-e2e-test resourceVersion: "19401" uid: 34b2619e-ec6a-46f7-9897-cb90f41adcb0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:40Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-e6f34-predictor-595bdccbdf namespace: kserve-ci-e2e-test resourceVersion: "15012" uid: 647a9119-b71c-4665-b33b-b89169512533 kind: Event lastTimestamp: "2026-04-20T20:15:40Z" message: 'Created pod: error-404-isvc-e6f34-predictor-595bdccbdf-lf8lw' metadata: creationTimestamp: "2026-04-20T20:15:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:15:40Z" name: error-404-isvc-e6f34-predictor-595bdccbdf.18a829ea9a0b7320 namespace: kserve-ci-e2e-test resourceVersion: "15034" uid: 21797dd3-30cd-41c4-8586-4e46133113a9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:40Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-e6f34-predictor namespace: kserve-ci-e2e-test resourceVersion: "15008" uid: cde4dbdb-f2e1-4966-a0f3-f42c455370fd kind: Event lastTimestamp: "2026-04-20T20:15:40Z" message: Scaled up replica set error-404-isvc-e6f34-predictor-595bdccbdf from 0 to 1 metadata: creationTimestamp: "2026-04-20T20:15:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:15:40Z" name: error-404-isvc-e6f34-predictor.18a829ea97435390 namespace: kserve-ci-e2e-test resourceVersion: "15016" uid: 267e6f02-3dbb-4217-b0f0-3238098f3719 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:15:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e6f34-predictor namespace: kserve-ci-e2e-test resourceVersion: "15011" uid: acc2a1db-6857-4fee-a962-c801224ab8e3 kind: Event lastTimestamp: "2026-04-20T20:16:10Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:15:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:16:10Z" name: error-404-isvc-e6f34-predictor.18a829ee15b3f94b namespace: kserve-ci-e2e-test resourceVersion: "15392" uid: 531e9baa-dbc3-40f5-a35e-f93500f7ee3a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:15:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e6f34-predictor namespace: kserve-ci-e2e-test resourceVersion: "15011" uid: acc2a1db-6857-4fee-a962-c801224ab8e3 kind: Event lastTimestamp: "2026-04-20T20:16:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:15:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:16:10Z" name: error-404-isvc-e6f34-predictor.18a829ee16193067 namespace: kserve-ci-e2e-test resourceVersion: "15393" uid: 768915c3-3d1b-4d36-ad9a-33311ba96ba3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:16:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e6f34-predictor namespace: kserve-ci-e2e-test resourceVersion: "15224" uid: acc2a1db-6857-4fee-a962-c801224ab8e3 kind: Event lastTimestamp: "2026-04-20T20:16:40Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:16:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:16:40Z" name: error-404-isvc-e6f34-predictor.18a829f5138b7145 namespace: kserve-ci-e2e-test resourceVersion: "15669" uid: b03befdb-0eb9-432d-a630-f1d2708fafd5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:16:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e6f34-predictor namespace: kserve-ci-e2e-test resourceVersion: "15224" uid: acc2a1db-6857-4fee-a962-c801224ab8e3 kind: Event lastTimestamp: "2026-04-20T20:16:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:16:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:16:40Z" name: error-404-isvc-e6f34-predictor.18a829f513f9e2b1 namespace: kserve-ci-e2e-test resourceVersion: "15670" uid: c9f5dec8-e351-4ef6-8ae3-d56dbdbc8258 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-20T20:15:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e6f34 namespace: kserve-ci-e2e-test resourceVersion: "14987" uid: 4f414bc4-6c37-4be6-a6a0-e184300a95e8 kind: Event lastTimestamp: "2026-04-20T20:16:28Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-20T20:15:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:16:28Z" name: error-404-isvc-e6f34.18a829ea96fabec8 namespace: kserve-ci-e2e-test resourceVersion: "15531" uid: 007505a0-bdcf-4858-a5b6-dbd61f6a0a19 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:16:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e6f34 namespace: kserve-ci-e2e-test resourceVersion: "15529" uid: 4f414bc4-6c37-4be6-a6a0-e184300a95e8 kind: Event lastTimestamp: "2026-04-20T20:16:28Z" message: InferenceService [error-404-isvc-e6f34] is Ready metadata: creationTimestamp: "2026-04-20T20:16:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:16:28Z" name: error-404-isvc-e6f34.18a829f5b5ab9fa9 namespace: kserve-ci-e2e-test resourceVersion: "15530" uid: ade7a39c-d2dd-4b6d-94d3-79c305f9c5f0 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-20T20:26:05.499023Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-fe410-predictor-6b76d4c7f7-bwcdp namespace: kserve-ci-e2e-test resourceVersion: "20307" uid: 9d52aa60-7277-4328-b3f1-c93fbd5c55a8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-fe410-predictor-6b76d4c7f7-bwcdp to ip-10-0-130-227.ec2.internal metadata: creationTimestamp: "2026-04-20T20:26:05Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-20T20:26:05Z" name: error-404-isvc-fe410-predictor-6b76d4c7f7-bwcdp.18a82a7c1a0cf8dc namespace: kserve-ci-e2e-test resourceVersion: "20311" uid: ff9a8916-36bf-4ef9-badb-8b9f9c8aa0e6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58dbb994fc-hbcnb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:26:05Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-fe410-predictor-6b76d4c7f7-bwcdp namespace: kserve-ci-e2e-test resourceVersion: "20309" uid: 9d52aa60-7277-4328-b3f1-c93fbd5c55a8 kind: Event lastTimestamp: "2026-04-20T20:26:05Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-fe410-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-20T20:26:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:26:05Z" name: error-404-isvc-fe410-predictor-6b76d4c7f7-bwcdp.18a82a7c2597c53d namespace: kserve-ci-e2e-test resourceVersion: "20319" uid: 167b2900-675d-4fad-a5b1-912b98016f70 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:26:06Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-fe410-predictor-6b76d4c7f7-bwcdp namespace: kserve-ci-e2e-test resourceVersion: "20316" uid: 9d52aa60-7277-4328-b3f1-c93fbd5c55a8 kind: Event lastTimestamp: "2026-04-20T20:26:06Z" message: Add eth0 [10.133.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-20T20:26:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-20T20:26:06Z" name: error-404-isvc-fe410-predictor-6b76d4c7f7-bwcdp.18a82a7c5702917b namespace: kserve-ci-e2e-test resourceVersion: "20340" uid: ec461fd8-e709-42f6-a36f-7b9835b4775a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:26:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fe410-predictor-6b76d4c7f7-bwcdp namespace: kserve-ci-e2e-test resourceVersion: "20309" uid: 9d52aa60-7277-4328-b3f1-c93fbd5c55a8 kind: Event lastTimestamp: "2026-04-20T20:26:06Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-20T20:26:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:26:06Z" name: error-404-isvc-fe410-predictor-6b76d4c7f7-bwcdp.18a82a7c58643bd6 namespace: kserve-ci-e2e-test resourceVersion: "20342" uid: 128fac94-64db-415b-9b0f-42dcc32e599c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:26:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fe410-predictor-6b76d4c7f7-bwcdp namespace: kserve-ci-e2e-test resourceVersion: "20309" uid: 9d52aa60-7277-4328-b3f1-c93fbd5c55a8 kind: Event lastTimestamp: "2026-04-20T20:26:06Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-20T20:26:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:26:06Z" name: error-404-isvc-fe410-predictor-6b76d4c7f7-bwcdp.18a82a7c5bb0ef80 namespace: kserve-ci-e2e-test resourceVersion: "20345" uid: 3897de22-0296-4320-8385-0954643e2f46 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:26:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fe410-predictor-6b76d4c7f7-bwcdp namespace: kserve-ci-e2e-test resourceVersion: "20309" uid: 9d52aa60-7277-4328-b3f1-c93fbd5c55a8 kind: Event lastTimestamp: "2026-04-20T20:26:06Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-20T20:26:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:26:06Z" name: error-404-isvc-fe410-predictor-6b76d4c7f7-bwcdp.18a82a7c5c1eb007 namespace: kserve-ci-e2e-test resourceVersion: "20346" uid: 4cea53f6-4577-4383-a249-cdacc7e626aa reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:26:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fe410-predictor-6b76d4c7f7-bwcdp namespace: kserve-ci-e2e-test resourceVersion: "20309" uid: 9d52aa60-7277-4328-b3f1-c93fbd5c55a8 kind: Event lastTimestamp: "2026-04-20T20:26:06Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-20T20:26:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:26:06Z" name: error-404-isvc-fe410-predictor-6b76d4c7f7-bwcdp.18a82a7c5c2aad41 namespace: kserve-ci-e2e-test resourceVersion: "20347" uid: 5d893d95-0f5b-4fd4-9d15-a7c9cdd212cc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:26:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fe410-predictor-6b76d4c7f7-bwcdp namespace: kserve-ci-e2e-test resourceVersion: "20309" uid: 9d52aa60-7277-4328-b3f1-c93fbd5c55a8 kind: Event lastTimestamp: "2026-04-20T20:26:06Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-20T20:26:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:26:06Z" name: error-404-isvc-fe410-predictor-6b76d4c7f7-bwcdp.18a82a7c5fb8f59f namespace: kserve-ci-e2e-test resourceVersion: "20348" uid: 673390a8-1238-4185-ba20-c44aa995f23a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:26:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fe410-predictor-6b76d4c7f7-bwcdp namespace: kserve-ci-e2e-test resourceVersion: "20309" uid: 9d52aa60-7277-4328-b3f1-c93fbd5c55a8 kind: Event lastTimestamp: "2026-04-20T20:26:06Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:26:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:26:06Z" name: error-404-isvc-fe410-predictor-6b76d4c7f7-bwcdp.18a82a7c60262f35 namespace: kserve-ci-e2e-test resourceVersion: "20349" uid: 633fa02c-6b75-43ad-b3ad-486088394137 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-20T20:26:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fe410-predictor-6b76d4c7f7-bwcdp namespace: kserve-ci-e2e-test resourceVersion: "20309" uid: 9d52aa60-7277-4328-b3f1-c93fbd5c55a8 kind: Event lastTimestamp: "2026-04-20T20:26:44Z" message: 'Readiness probe failed: dial tcp 10.133.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-20T20:26:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:26:44Z" name: error-404-isvc-fe410-predictor-6b76d4c7f7-bwcdp.18a82a7cc4408cb9 namespace: kserve-ci-e2e-test resourceVersion: "20725" uid: a18a83a3-2b02-4dfc-b765-1b15d5cb05c7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fe410-predictor-6b76d4c7f7-bwcdp namespace: kserve-ci-e2e-test resourceVersion: "20309" uid: 9d52aa60-7277-4328-b3f1-c93fbd5c55a8 kind: Event lastTimestamp: "2026-04-20T20:35:20Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-20T20:35:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:35:20Z" name: error-404-isvc-fe410-predictor-6b76d4c7f7-bwcdp.18a82afd4b8a475f namespace: kserve-ci-e2e-test resourceVersion: "24575" uid: 823e82b4-594f-409b-b320-bfbd10e6181b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fe410-predictor-6b76d4c7f7-bwcdp namespace: kserve-ci-e2e-test resourceVersion: "20309" uid: 9d52aa60-7277-4328-b3f1-c93fbd5c55a8 kind: Event lastTimestamp: "2026-04-20T20:35:20Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:35:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:35:20Z" name: error-404-isvc-fe410-predictor-6b76d4c7f7-bwcdp.18a82afd4b8b5a39 namespace: kserve-ci-e2e-test resourceVersion: "24578" uid: de2523d8-a4e2-4e5d-9934-43fd7ab6db51 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-227.ec2.internal source: component: kubelet host: ip-10-0-130-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:26:05Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-fe410-predictor-6b76d4c7f7 namespace: kserve-ci-e2e-test resourceVersion: "20300" uid: 8f8db89f-01bc-4c02-afc0-d870b1f2d467 kind: Event lastTimestamp: "2026-04-20T20:26:05Z" message: 'Created pod: error-404-isvc-fe410-predictor-6b76d4c7f7-bwcdp' metadata: creationTimestamp: "2026-04-20T20:26:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:26:05Z" name: error-404-isvc-fe410-predictor-6b76d4c7f7.18a82a7c197c2061 namespace: kserve-ci-e2e-test resourceVersion: "20308" uid: 5f507a4e-2792-4db5-bdaa-a890c6b2f47d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:26:05Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-fe410-predictor namespace: kserve-ci-e2e-test resourceVersion: "20296" uid: ac6858d2-c3f1-4e8e-a149-2809c084fd4c kind: Event lastTimestamp: "2026-04-20T20:26:05Z" message: Scaled up replica set error-404-isvc-fe410-predictor-6b76d4c7f7 from 0 to 1 metadata: creationTimestamp: "2026-04-20T20:26:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:26:05Z" name: error-404-isvc-fe410-predictor.18a82a7c17dbaa54 namespace: kserve-ci-e2e-test resourceVersion: "20302" uid: f851f1f3-c0ac-42f6-af25-dd2921b70005 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:26:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fe410-predictor namespace: kserve-ci-e2e-test resourceVersion: "20299" uid: 8f5b22cd-21ed-4ec9-bdfb-40dd0802d816 kind: Event lastTimestamp: "2026-04-20T20:26:35Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:26:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:26:35Z" name: error-404-isvc-fe410-predictor.18a82a7f965a9294 namespace: kserve-ci-e2e-test resourceVersion: "20657" uid: 7ac701c7-984c-4baa-8ab6-05a6f89559ea reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:26:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fe410-predictor namespace: kserve-ci-e2e-test resourceVersion: "20299" uid: 8f5b22cd-21ed-4ec9-bdfb-40dd0802d816 kind: Event lastTimestamp: "2026-04-20T20:26:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:26:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:26:35Z" name: error-404-isvc-fe410-predictor.18a82a7f96bd9409 namespace: kserve-ci-e2e-test resourceVersion: "20658" uid: b8e66bc5-be45-455a-9050-14e7040d7d9a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-20T20:26:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fe410-predictor namespace: kserve-ci-e2e-test resourceVersion: "20488" uid: 8f5b22cd-21ed-4ec9-bdfb-40dd0802d816 kind: Event lastTimestamp: "2026-04-20T20:27:20Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:26:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:27:20Z" name: error-404-isvc-fe410-predictor.18a82a869418a46d namespace: kserve-ci-e2e-test resourceVersion: "21051" uid: 7b01cb52-6503-4cf3-a7ae-f87598d01366 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-20T20:26:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fe410-predictor namespace: kserve-ci-e2e-test resourceVersion: "20488" uid: 8f5b22cd-21ed-4ec9-bdfb-40dd0802d816 kind: Event lastTimestamp: "2026-04-20T20:27:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:26:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:27:20Z" name: error-404-isvc-fe410-predictor.18a82a8694774565 namespace: kserve-ci-e2e-test resourceVersion: "21052" uid: f29f07c8-d511-4f68-bf27-c1ef2e63859a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-20T20:26:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fe410 namespace: kserve-ci-e2e-test resourceVersion: "20266" uid: 46464996-50aa-4216-9d44-c5badf897bd2 kind: Event lastTimestamp: "2026-04-20T20:26:54Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-20T20:26:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:26:54Z" name: error-404-isvc-fe410.18a82a7c17aca6e1 namespace: kserve-ci-e2e-test resourceVersion: "20816" uid: b0bf3f10-d862-4265-a935-e17dc99eb87f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:26:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fe410 namespace: kserve-ci-e2e-test resourceVersion: "20266" uid: 46464996-50aa-4216-9d44-c5badf897bd2 kind: Event lastTimestamp: "2026-04-20T20:26:05Z" message: 'Failed to update status for InferenceService "error-404-isvc-fe410": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-fe410": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-20T20:26:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:26:05Z" name: error-404-isvc-fe410.18a82a7c18135263 namespace: kserve-ci-e2e-test resourceVersion: "20304" uid: 8287bc03-c45c-4506-b0e6-1a47355578b1 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:26:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fe410 namespace: kserve-ci-e2e-test resourceVersion: "20266" uid: 46464996-50aa-4216-9d44-c5badf897bd2 kind: Event lastTimestamp: "2026-04-20T20:26:05Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-fe410": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-20T20:26:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:26:05Z" name: error-404-isvc-fe410.18a82a7c18139e6c namespace: kserve-ci-e2e-test resourceVersion: "20305" uid: c814e986-0efe-43a3-82d4-a4c6200c9e82 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:26:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fe410 namespace: kserve-ci-e2e-test resourceVersion: "20812" uid: 46464996-50aa-4216-9d44-c5badf897bd2 kind: Event lastTimestamp: "2026-04-20T20:26:54Z" message: InferenceService [error-404-isvc-fe410] is Ready metadata: creationTimestamp: "2026-04-20T20:26:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:26:54Z" name: error-404-isvc-fe410.18a82a877f241e49 namespace: kserve-ci-e2e-test resourceVersion: "20813" uid: 5584d996-1e17-4d0b-8623-e40f295b569b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-20T20:13:30.064506Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-7cl29 namespace: kserve-ci-e2e-test resourceVersion: "13369" uid: d251e1b5-ef06-472e-a8d0-6d5641b1a9e6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-5b497dcd98-7cl29 to ip-10-0-140-19.ec2.internal metadata: creationTimestamp: "2026-04-20T20:13:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-20T20:13:30Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-7cl29.18a829cc36a50e81 namespace: kserve-ci-e2e-test resourceVersion: "13375" uid: 59832a52-2488-46b5-ae5b-d9d5d073e78c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58dbb994fc-hbcnb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:13:30Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-7cl29 namespace: kserve-ci-e2e-test resourceVersion: "13388" uid: d251e1b5-ef06-472e-a8d0-6d5641b1a9e6 kind: Event lastTimestamp: "2026-04-20T20:13:30Z" message: Add eth0 [10.132.0.17/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-20T20:13:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-20T20:13:30Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-7cl29.18a829cc516068dc namespace: kserve-ci-e2e-test resourceVersion: "13459" uid: d5c01761-b48a-4178-abb1-7f2c4c7a0194 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:13:30Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-7cl29 namespace: kserve-ci-e2e-test resourceVersion: "13372" uid: d251e1b5-ef06-472e-a8d0-6d5641b1a9e6 kind: Event lastTimestamp: "2026-04-20T20:13:30Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:0027065b33c40f52a3d9fc655cbe89bc4c54c77f7f755275def0561338196f29" metadata: creationTimestamp: "2026-04-20T20:13:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:13:30Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-7cl29.18a829cc52b5129e namespace: kserve-ci-e2e-test resourceVersion: "13461" uid: 95439d78-2a9b-46d1-8500-8c67bbac1025 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:13:44Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-7cl29 namespace: kserve-ci-e2e-test resourceVersion: "13372" uid: d251e1b5-ef06-472e-a8d0-6d5641b1a9e6 kind: Event lastTimestamp: "2026-04-20T20:13:44Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:0027065b33c40f52a3d9fc655cbe89bc4c54c77f7f755275def0561338196f29" in 14.359s (14.359s including waiting). Image size: 300951977 bytes.' metadata: creationTimestamp: "2026-04-20T20:13:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:13:44Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-7cl29.18a829cfaa9be735 namespace: kserve-ci-e2e-test resourceVersion: "13637" uid: 7168eaa3-4d5b-41d4-a6a8-b9374a8395a8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:13:44Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-7cl29 namespace: kserve-ci-e2e-test resourceVersion: "13372" uid: d251e1b5-ef06-472e-a8d0-6d5641b1a9e6 kind: Event lastTimestamp: "2026-04-20T20:13:44Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-20T20:13:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:13:44Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-7cl29.18a829cfae330b25 namespace: kserve-ci-e2e-test resourceVersion: "13639" uid: 51febf94-b2b0-4aa8-9868-2fbe3961aa5a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:13:44Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-7cl29 namespace: kserve-ci-e2e-test resourceVersion: "13372" uid: d251e1b5-ef06-472e-a8d0-6d5641b1a9e6 kind: Event lastTimestamp: "2026-04-20T20:13:44Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-20T20:13:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:13:44Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-7cl29.18a829cfaeb231b3 namespace: kserve-ci-e2e-test resourceVersion: "13641" uid: 94d5bcb9-bd4f-487b-b921-54ae5e65a192 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:13:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-7cl29 namespace: kserve-ci-e2e-test resourceVersion: "13372" uid: d251e1b5-ef06-472e-a8d0-6d5641b1a9e6 kind: Event lastTimestamp: "2026-04-20T20:13:49Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1293" metadata: creationTimestamp: "2026-04-20T20:13:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:13:49Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-7cl29.18a829d0b2393122 namespace: kserve-ci-e2e-test resourceVersion: "13720" uid: 2e4779f4-4804-4fea-b28c-ead18720e5fd reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:13:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-7cl29 namespace: kserve-ci-e2e-test resourceVersion: "13372" uid: d251e1b5-ef06-472e-a8d0-6d5641b1a9e6 kind: Event lastTimestamp: "2026-04-20T20:13:56Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1293" in 7.503s (7.503s including waiting). Image size: 1560610732 bytes.' metadata: creationTimestamp: "2026-04-20T20:13:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:13:56Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-7cl29.18a829d27170949c namespace: kserve-ci-e2e-test resourceVersion: "13790" uid: d8ef297b-e672-4780-8bcd-7468ff93bfe9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:13:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-7cl29 namespace: kserve-ci-e2e-test resourceVersion: "13372" uid: d251e1b5-ef06-472e-a8d0-6d5641b1a9e6 kind: Event lastTimestamp: "2026-04-20T20:13:56Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-20T20:13:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:13:56Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-7cl29.18a829d274917bb8 namespace: kserve-ci-e2e-test resourceVersion: "13793" uid: 670d58f9-cbb7-4fde-bb17-dedc5af52c27 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:13:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-7cl29 namespace: kserve-ci-e2e-test resourceVersion: "13372" uid: d251e1b5-ef06-472e-a8d0-6d5641b1a9e6 kind: Event lastTimestamp: "2026-04-20T20:13:56Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-20T20:13:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:13:56Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-7cl29.18a829d27521b322 namespace: kserve-ci-e2e-test resourceVersion: "13796" uid: 27587860-7ad1-4838-95fd-7003938970f1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:13:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-7cl29 namespace: kserve-ci-e2e-test resourceVersion: "13372" uid: d251e1b5-ef06-472e-a8d0-6d5641b1a9e6 kind: Event lastTimestamp: "2026-04-20T20:13:56Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-20T20:13:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:13:56Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-7cl29.18a829d2752dc30e namespace: kserve-ci-e2e-test resourceVersion: "13797" uid: 3bf8deeb-fd30-447a-abd4-49456c35eddb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:13:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-7cl29 namespace: kserve-ci-e2e-test resourceVersion: "13372" uid: d251e1b5-ef06-472e-a8d0-6d5641b1a9e6 kind: Event lastTimestamp: "2026-04-20T20:13:56Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-20T20:13:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:13:56Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-7cl29.18a829d27859e7ea namespace: kserve-ci-e2e-test resourceVersion: "13800" uid: 6b7ecb21-3eb1-457c-a719-e2c0f34b43f0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:13:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-7cl29 namespace: kserve-ci-e2e-test resourceVersion: "13372" uid: d251e1b5-ef06-472e-a8d0-6d5641b1a9e6 kind: Event lastTimestamp: "2026-04-20T20:13:56Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:13:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:13:56Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-7cl29.18a829d278cf5113 namespace: kserve-ci-e2e-test resourceVersion: "13802" uid: 43b0c612-206c-4684-940e-4a48b7802d41 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-20T20:13:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-7cl29 namespace: kserve-ci-e2e-test resourceVersion: "13372" uid: d251e1b5-ef06-472e-a8d0-6d5641b1a9e6 kind: Event lastTimestamp: "2026-04-20T20:15:43Z" message: 'Readiness probe failed: dial tcp 10.132.0.17:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-20T20:13:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:15:43Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-7cl29.18a829d290ba00cb namespace: kserve-ci-e2e-test resourceVersion: "15103" uid: 99f3f5c7-4071-4f30-a032-55ef020dad91 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-7cl29 namespace: kserve-ci-e2e-test resourceVersion: "13372" uid: d251e1b5-ef06-472e-a8d0-6d5641b1a9e6 kind: Event lastTimestamp: "2026-04-20T20:15:40Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-20T20:15:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:15:40Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-7cl29.18a829ea91d70fb0 namespace: kserve-ci-e2e-test resourceVersion: "14990" uid: 65e80e54-9d76-418e-9f59-c5dce64eb535 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-7cl29 namespace: kserve-ci-e2e-test resourceVersion: "13372" uid: d251e1b5-ef06-472e-a8d0-6d5641b1a9e6 kind: Event lastTimestamp: "2026-04-20T20:15:40Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:15:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:15:40Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-7cl29.18a829ea91d786d3 namespace: kserve-ci-e2e-test resourceVersion: "14991" uid: 33f7e380-65e8-448e-85e1-9b2dd6112f9b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-7cl29 namespace: kserve-ci-e2e-test resourceVersion: "13372" uid: d251e1b5-ef06-472e-a8d0-6d5641b1a9e6 kind: Event lastTimestamp: "2026-04-20T20:15:43Z" message: 'Readiness probe failed: Get "https://10.132.0.17:8643/healthz": dial tcp 10.132.0.17:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-20T20:15:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:15:43Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-7cl29.18a829eb3efee1e5 namespace: kserve-ci-e2e-test resourceVersion: "15101" uid: 0eaeb4d0-5103-416b-b10d-c02a02779943 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:13:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-5b497dcd98 namespace: kserve-ci-e2e-test resourceVersion: "13356" uid: 6c552591-93bc-4f10-a5d2-7654fe11b7b9 kind: Event lastTimestamp: "2026-04-20T20:13:30Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-5b497dcd98-7cl29' metadata: creationTimestamp: "2026-04-20T20:13:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:13:30Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98.18a829cc3631d2d2 namespace: kserve-ci-e2e-test resourceVersion: "13371" uid: cd5e8a02-3781-4080-9510-0d84ed210c92 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:13:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13355" uid: 804d58a5-9195-4c7f-8e42-fda964360f00 kind: Event lastTimestamp: "2026-04-20T20:13:29Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-5b497dcd98 from 0 to 1 metadata: creationTimestamp: "2026-04-20T20:13:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:13:29Z" name: isvc-sklearn-graph-1-predictor.18a829cc2e6acaea namespace: kserve-ci-e2e-test resourceVersion: "13357" uid: 680c468f-d33a-41e8-b7bd-5a09f5549219 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-20T20:13:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13363" uid: 13882052-6880-470c-91f1-184151ad2d53 kind: Event lastTimestamp: "2026-04-20T20:14:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:13:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:14:30Z" name: isvc-sklearn-graph-1-predictor.18a829cfb4142018 namespace: kserve-ci-e2e-test resourceVersion: "14085" uid: 01388843-c4f3-4d1e-b50b-62687093c09a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-20T20:13:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13363" uid: 13882052-6880-470c-91f1-184151ad2d53 kind: Event lastTimestamp: "2026-04-20T20:14:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:13:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:14:30Z" name: isvc-sklearn-graph-1-predictor.18a829cfb47ad5d8 namespace: kserve-ci-e2e-test resourceVersion: "14087" uid: a1262248-43b9-47bc-8a12-d486c622cc68 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-20T20:14:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13648" uid: 13882052-6880-470c-91f1-184151ad2d53 kind: Event lastTimestamp: "2026-04-20T20:15:30Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:14:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:15:30Z" name: isvc-sklearn-graph-1-predictor.18a829ddb0f1ba68 namespace: kserve-ci-e2e-test resourceVersion: "14813" uid: 6b07e20b-be3f-49fb-ad83-f2a99b67e762 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-20T20:14:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13648" uid: 13882052-6880-470c-91f1-184151ad2d53 kind: Event lastTimestamp: "2026-04-20T20:15:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:14:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:15:30Z" name: isvc-sklearn-graph-1-predictor.18a829ddb160ac8c namespace: kserve-ci-e2e-test resourceVersion: "14814" uid: 6066ce16-29ac-4f01-84e3-54714a4427d5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-20T20:13:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "13340" uid: 4d7e4771-1ebf-421f-a7b5-60c5395eba17 kind: Event lastTimestamp: "2026-04-20T20:15:03Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-20T20:13:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:15:03Z" name: isvc-sklearn-graph-1.18a829cc34a2b88a namespace: kserve-ci-e2e-test resourceVersion: "14564" uid: 363ce93e-4f28-4173-b6dc-1f1de85ff2fc reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14561" uid: 4d7e4771-1ebf-421f-a7b5-60c5395eba17 kind: Event lastTimestamp: "2026-04-20T20:15:03Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-04-20T20:15:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:15:03Z" name: isvc-sklearn-graph-1.18a829e1f5d04d3c namespace: kserve-ci-e2e-test resourceVersion: "14562" uid: 8dc7660a-c5e4-4243-95bf-ce636bb5709f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-20T20:13:31.153221Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-pr4dz namespace: kserve-ci-e2e-test resourceVersion: "13491" uid: dabf688b-196b-4f63-9154-7b6bb4fc446c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-847f6cf74b-pr4dz to ip-10-0-140-19.ec2.internal metadata: creationTimestamp: "2026-04-20T20:13:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-20T20:13:31Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-pr4dz.18a829cc77898ffb namespace: kserve-ci-e2e-test resourceVersion: "13495" uid: 310b305e-82d5-41f5-b90c-d6d7803d2381 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58dbb994fc-hbcnb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:13:31Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-pr4dz namespace: kserve-ci-e2e-test resourceVersion: "13499" uid: dabf688b-196b-4f63-9154-7b6bb4fc446c kind: Event lastTimestamp: "2026-04-20T20:13:31Z" message: Add eth0 [10.132.0.19/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-20T20:13:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-20T20:13:31Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-pr4dz.18a829cc924e6b26 namespace: kserve-ci-e2e-test resourceVersion: "13509" uid: 39501786-48f9-4cb8-af25-bb2f9de0ea5b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:13:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-pr4dz namespace: kserve-ci-e2e-test resourceVersion: "13493" uid: dabf688b-196b-4f63-9154-7b6bb4fc446c kind: Event lastTimestamp: "2026-04-20T20:13:31Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:0027065b33c40f52a3d9fc655cbe89bc4c54c77f7f755275def0561338196f29" metadata: creationTimestamp: "2026-04-20T20:13:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:13:31Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-pr4dz.18a829cc93ad787d namespace: kserve-ci-e2e-test resourceVersion: "13511" uid: c67d8cac-04a4-424e-a9ef-27aad0bcbc15 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:13:44Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-pr4dz namespace: kserve-ci-e2e-test resourceVersion: "13493" uid: dabf688b-196b-4f63-9154-7b6bb4fc446c kind: Event lastTimestamp: "2026-04-20T20:13:44Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:0027065b33c40f52a3d9fc655cbe89bc4c54c77f7f755275def0561338196f29" in 13.269s (13.269s including waiting). Image size: 300951977 bytes.' metadata: creationTimestamp: "2026-04-20T20:13:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:13:44Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-pr4dz.18a829cfaa9beccd namespace: kserve-ci-e2e-test resourceVersion: "13636" uid: d56221f5-7c93-408a-bd74-39635e89691e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:13:44Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-pr4dz namespace: kserve-ci-e2e-test resourceVersion: "13493" uid: dabf688b-196b-4f63-9154-7b6bb4fc446c kind: Event lastTimestamp: "2026-04-20T20:13:44Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-20T20:13:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:13:44Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-pr4dz.18a829cfae3f4309 namespace: kserve-ci-e2e-test resourceVersion: "13640" uid: 5772c9e5-49a1-4662-8efb-6dd54b7b9d98 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:13:44Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-pr4dz namespace: kserve-ci-e2e-test resourceVersion: "13493" uid: dabf688b-196b-4f63-9154-7b6bb4fc446c kind: Event lastTimestamp: "2026-04-20T20:13:44Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-20T20:13:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:13:44Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-pr4dz.18a829cfaebfe979 namespace: kserve-ci-e2e-test resourceVersion: "13642" uid: c0840d14-1626-40f3-82ec-9ac4711f73ad reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:13:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-pr4dz namespace: kserve-ci-e2e-test resourceVersion: "13493" uid: dabf688b-196b-4f63-9154-7b6bb4fc446c kind: Event lastTimestamp: "2026-04-20T20:13:49Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1293" metadata: creationTimestamp: "2026-04-20T20:13:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:13:49Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-pr4dz.18a829d0b2221ee9 namespace: kserve-ci-e2e-test resourceVersion: "13717" uid: 67387193-d6c5-463a-9773-55677ad9301c reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:13:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-pr4dz namespace: kserve-ci-e2e-test resourceVersion: "13493" uid: dabf688b-196b-4f63-9154-7b6bb4fc446c kind: Event lastTimestamp: "2026-04-20T20:13:56Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1293" in 7.491s (7.491s including waiting). Image size: 1560610732 bytes.' metadata: creationTimestamp: "2026-04-20T20:13:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:13:56Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-pr4dz.18a829d270aa30ce namespace: kserve-ci-e2e-test resourceVersion: "13789" uid: 3ac0b154-ab74-4594-a3b8-233f418978ff reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:13:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-pr4dz namespace: kserve-ci-e2e-test resourceVersion: "13493" uid: dabf688b-196b-4f63-9154-7b6bb4fc446c kind: Event lastTimestamp: "2026-04-20T20:13:56Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-20T20:13:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:13:56Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-pr4dz.18a829d27484d152 namespace: kserve-ci-e2e-test resourceVersion: "13792" uid: bf56fadd-3bb7-428e-9039-86c235f1b653 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:13:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-pr4dz namespace: kserve-ci-e2e-test resourceVersion: "13493" uid: dabf688b-196b-4f63-9154-7b6bb4fc446c kind: Event lastTimestamp: "2026-04-20T20:13:56Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-20T20:13:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:13:56Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-pr4dz.18a829d2750b7d57 namespace: kserve-ci-e2e-test resourceVersion: "13794" uid: 00ccbcf5-ef97-417c-af5b-1a88689370b5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:13:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-pr4dz namespace: kserve-ci-e2e-test resourceVersion: "13493" uid: dabf688b-196b-4f63-9154-7b6bb4fc446c kind: Event lastTimestamp: "2026-04-20T20:13:56Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-20T20:13:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:13:56Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-pr4dz.18a829d27516700b namespace: kserve-ci-e2e-test resourceVersion: "13795" uid: fcc533a9-4b47-4532-8eeb-e4721b8546e9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:13:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-pr4dz namespace: kserve-ci-e2e-test resourceVersion: "13493" uid: dabf688b-196b-4f63-9154-7b6bb4fc446c kind: Event lastTimestamp: "2026-04-20T20:13:56Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-20T20:13:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:13:56Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-pr4dz.18a829d2784e45d9 namespace: kserve-ci-e2e-test resourceVersion: "13799" uid: 34cd33c5-d89b-4439-9b33-2bfe461911d3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:13:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-pr4dz namespace: kserve-ci-e2e-test resourceVersion: "13493" uid: dabf688b-196b-4f63-9154-7b6bb4fc446c kind: Event lastTimestamp: "2026-04-20T20:13:56Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:13:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:13:56Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-pr4dz.18a829d278c1fdc0 namespace: kserve-ci-e2e-test resourceVersion: "13801" uid: 71f30478-0d12-4bd0-96b9-fa4840a7a6c7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-20T20:13:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-pr4dz namespace: kserve-ci-e2e-test resourceVersion: "13493" uid: dabf688b-196b-4f63-9154-7b6bb4fc446c kind: Event lastTimestamp: "2026-04-20T20:15:43Z" message: 'Readiness probe failed: dial tcp 10.132.0.19:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-20T20:13:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:15:43Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-pr4dz.18a829d290a54c5d namespace: kserve-ci-e2e-test resourceVersion: "15104" uid: 37dfa97d-21ee-48fb-9d3a-741109706b8a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-pr4dz namespace: kserve-ci-e2e-test resourceVersion: "13493" uid: dabf688b-196b-4f63-9154-7b6bb4fc446c kind: Event lastTimestamp: "2026-04-20T20:15:40Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-20T20:15:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:15:40Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-pr4dz.18a829ea8a793f23 namespace: kserve-ci-e2e-test resourceVersion: "14961" uid: d2fb0aa8-13ae-4e26-8e19-a85808d0011b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-pr4dz namespace: kserve-ci-e2e-test resourceVersion: "13493" uid: dabf688b-196b-4f63-9154-7b6bb4fc446c kind: Event lastTimestamp: "2026-04-20T20:15:40Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:15:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:15:40Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-pr4dz.18a829ea8a79e8bc namespace: kserve-ci-e2e-test resourceVersion: "14966" uid: 629b2a06-1457-4dab-a195-37aa7acec4d7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-pr4dz namespace: kserve-ci-e2e-test resourceVersion: "13493" uid: dabf688b-196b-4f63-9154-7b6bb4fc446c kind: Event lastTimestamp: "2026-04-20T20:15:43Z" message: 'Readiness probe failed: Get "https://10.132.0.19:8643/healthz": dial tcp 10.132.0.19:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-20T20:15:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:15:43Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-pr4dz.18a829eb3f001570 namespace: kserve-ci-e2e-test resourceVersion: "15102" uid: cf100321-b2e9-4bf2-ad60-f69c1dc93a03 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:13:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-847f6cf74b namespace: kserve-ci-e2e-test resourceVersion: "13477" uid: b2b56132-6c36-4d92-abbd-928e61976f32 kind: Event lastTimestamp: "2026-04-20T20:13:31Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-847f6cf74b-pr4dz' metadata: creationTimestamp: "2026-04-20T20:13:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:13:31Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b.18a829cc7721a11a namespace: kserve-ci-e2e-test resourceVersion: "13492" uid: ea4383af-4115-4064-b9d4-f4a52ee085f7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:13:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13475" uid: 0e4cc252-cc3f-421f-8fec-7c3be728b62f kind: Event lastTimestamp: "2026-04-20T20:13:30Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-847f6cf74b from 0 to 1 metadata: creationTimestamp: "2026-04-20T20:13:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:13:30Z" name: isvc-sklearn-graph-2-predictor.18a829cc6112c220 namespace: kserve-ci-e2e-test resourceVersion: "13478" uid: 6a1e2684-f1b8-4343-bedb-16395c7b594c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-20T20:13:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13476" uid: e480f226-0d93-4522-bb2b-26a827c7e043 kind: Event lastTimestamp: "2026-04-20T20:14:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:13:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:14:30Z" name: isvc-sklearn-graph-2-predictor.18a829cfdf92fbf6 namespace: kserve-ci-e2e-test resourceVersion: "14098" uid: 5615a6cb-987c-4050-9355-07217a494c9c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-20T20:13:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13476" uid: e480f226-0d93-4522-bb2b-26a827c7e043 kind: Event lastTimestamp: "2026-04-20T20:14:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:13:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:14:30Z" name: isvc-sklearn-graph-2-predictor.18a829cfdffbd2b3 namespace: kserve-ci-e2e-test resourceVersion: "14099" uid: 21491a27-15a1-424d-bc3d-17f3dde8228b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-20T20:14:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13676" uid: e480f226-0d93-4522-bb2b-26a827c7e043 kind: Event lastTimestamp: "2026-04-20T20:15:30Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:14:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:15:30Z" name: isvc-sklearn-graph-2-predictor.18a829dddaf5c085 namespace: kserve-ci-e2e-test resourceVersion: "14835" uid: b0ece02a-c0d0-42b2-9b7e-42cf3ebe623a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-20T20:14:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13676" uid: e480f226-0d93-4522-bb2b-26a827c7e043 kind: Event lastTimestamp: "2026-04-20T20:15:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:14:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:15:30Z" name: isvc-sklearn-graph-2-predictor.18a829dddb56eb18 namespace: kserve-ci-e2e-test resourceVersion: "14836" uid: 5e27a217-21c6-4846-ad48-09e828128eba reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-20T20:13:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "13456" uid: 58a62dab-6415-4f26-8d23-c5a20ccaea7e kind: Event lastTimestamp: "2026-04-20T20:15:03Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-20T20:13:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:15:03Z" name: isvc-sklearn-graph-2.18a829cc60f426ad namespace: kserve-ci-e2e-test resourceVersion: "14563" uid: 9e127a55-f877-4b5a-abcb-353751eca03e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14558" uid: 58a62dab-6415-4f26-8d23-c5a20ccaea7e kind: Event lastTimestamp: "2026-04-20T20:15:03Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-04-20T20:15:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:15:03Z" name: isvc-sklearn-graph-2.18a829e1f3cb1240 namespace: kserve-ci-e2e-test resourceVersion: "14559" uid: d80c18b2-3b91-4664-a6b9-d0fdb39f11e4 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-20T20:13:30.259019Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-52hdf namespace: kserve-ci-e2e-test resourceVersion: "13423" uid: e927b850-b5b5-4cea-862e-4e4d1d93c247 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-669d8d6456-52hdf to ip-10-0-131-234.ec2.internal metadata: creationTimestamp: "2026-04-20T20:13:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-20T20:13:30Z" name: isvc-xgboost-graph-predictor-669d8d6456-52hdf.18a829cc423d1784 namespace: kserve-ci-e2e-test resourceVersion: "13427" uid: f2703a1d-18e9-432d-a9ed-d246e7383431 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58dbb994fc-hbcnb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:13:30Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-52hdf namespace: kserve-ci-e2e-test resourceVersion: "13425" uid: e927b850-b5b5-4cea-862e-4e4d1d93c247 kind: Event lastTimestamp: "2026-04-20T20:13:30Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-20T20:13:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:13:30Z" name: isvc-xgboost-graph-predictor-669d8d6456-52hdf.18a829cc4f4d6051 namespace: kserve-ci-e2e-test resourceVersion: "13458" uid: a2296a30-f330-4db8-9ca3-3a5c20f52c51 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:13:31Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-52hdf namespace: kserve-ci-e2e-test resourceVersion: "13432" uid: e927b850-b5b5-4cea-862e-4e4d1d93c247 kind: Event lastTimestamp: "2026-04-20T20:13:31Z" message: Add eth0 [10.134.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-20T20:13:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-20T20:13:31Z" name: isvc-xgboost-graph-predictor-669d8d6456-52hdf.18a829cc7f6b0000 namespace: kserve-ci-e2e-test resourceVersion: "13504" uid: 17863b58-35ba-4957-9a87-79056b73a332 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:13:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-52hdf namespace: kserve-ci-e2e-test resourceVersion: "13425" uid: e927b850-b5b5-4cea-862e-4e4d1d93c247 kind: Event lastTimestamp: "2026-04-20T20:13:31Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:0027065b33c40f52a3d9fc655cbe89bc4c54c77f7f755275def0561338196f29" metadata: creationTimestamp: "2026-04-20T20:13:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:13:31Z" name: isvc-xgboost-graph-predictor-669d8d6456-52hdf.18a829cc80da6c4b namespace: kserve-ci-e2e-test resourceVersion: "13507" uid: 8ef58056-c356-44cf-8b09-900612df6d08 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:13:35Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-52hdf namespace: kserve-ci-e2e-test resourceVersion: "13425" uid: e927b850-b5b5-4cea-862e-4e4d1d93c247 kind: Event lastTimestamp: "2026-04-20T20:13:35Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:0027065b33c40f52a3d9fc655cbe89bc4c54c77f7f755275def0561338196f29" in 3.741s (3.741s including waiting). Image size: 300951977 bytes.' metadata: creationTimestamp: "2026-04-20T20:13:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:13:35Z" name: isvc-xgboost-graph-predictor-669d8d6456-52hdf.18a829cd5fe37c80 namespace: kserve-ci-e2e-test resourceVersion: "13555" uid: e0cac7c5-e5e6-40d4-a5e7-bc7c3858f585 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:13:35Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-52hdf namespace: kserve-ci-e2e-test resourceVersion: "13425" uid: e927b850-b5b5-4cea-862e-4e4d1d93c247 kind: Event lastTimestamp: "2026-04-20T20:13:35Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-20T20:13:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:13:35Z" name: isvc-xgboost-graph-predictor-669d8d6456-52hdf.18a829cd630d4a82 namespace: kserve-ci-e2e-test resourceVersion: "13556" uid: 87cccef9-a9da-48e6-ad85-0db4e749d8cc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:13:35Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-52hdf namespace: kserve-ci-e2e-test resourceVersion: "13425" uid: e927b850-b5b5-4cea-862e-4e4d1d93c247 kind: Event lastTimestamp: "2026-04-20T20:13:35Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-20T20:13:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:13:35Z" name: isvc-xgboost-graph-predictor-669d8d6456-52hdf.18a829cd6383548d namespace: kserve-ci-e2e-test resourceVersion: "13557" uid: c7b9e525-a3b2-4c17-ad86-915555e7e1fc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:13:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-52hdf namespace: kserve-ci-e2e-test resourceVersion: "13425" uid: e927b850-b5b5-4cea-862e-4e4d1d93c247 kind: Event lastTimestamp: "2026-04-20T20:13:39Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-04-20T20:13:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:13:40Z" name: isvc-xgboost-graph-predictor-669d8d6456-52hdf.18a829ce86d763c6 namespace: kserve-ci-e2e-test resourceVersion: "13597" uid: 6efa39b8-7c37-4b8a-9317-113fc9a4a8b0 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:14:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-52hdf namespace: kserve-ci-e2e-test resourceVersion: "13425" uid: e927b850-b5b5-4cea-862e-4e4d1d93c247 kind: Event lastTimestamp: "2026-04-20T20:14:06Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 26.11s (26.11s including waiting). Image size: 1306414331 bytes.' metadata: creationTimestamp: "2026-04-20T20:14:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:14:06Z" name: isvc-xgboost-graph-predictor-669d8d6456-52hdf.18a829d49b21cb9f namespace: kserve-ci-e2e-test resourceVersion: "13890" uid: 6654ab80-5f26-49d2-ac72-fd3887793d8c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:14:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-52hdf namespace: kserve-ci-e2e-test resourceVersion: "13425" uid: e927b850-b5b5-4cea-862e-4e4d1d93c247 kind: Event lastTimestamp: "2026-04-20T20:14:06Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-20T20:14:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:14:06Z" name: isvc-xgboost-graph-predictor-669d8d6456-52hdf.18a829d49ee3c51c namespace: kserve-ci-e2e-test resourceVersion: "13892" uid: b565af12-1adc-4a05-8fa5-586fcbd78d5b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:14:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-52hdf namespace: kserve-ci-e2e-test resourceVersion: "13425" uid: e927b850-b5b5-4cea-862e-4e4d1d93c247 kind: Event lastTimestamp: "2026-04-20T20:14:06Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-20T20:14:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:14:06Z" name: isvc-xgboost-graph-predictor-669d8d6456-52hdf.18a829d49f5a2fc7 namespace: kserve-ci-e2e-test resourceVersion: "13893" uid: eece9e37-4011-4333-bb73-f2d466895bdc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:14:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-52hdf namespace: kserve-ci-e2e-test resourceVersion: "13425" uid: e927b850-b5b5-4cea-862e-4e4d1d93c247 kind: Event lastTimestamp: "2026-04-20T20:14:06Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-04-20T20:14:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:14:06Z" name: isvc-xgboost-graph-predictor-669d8d6456-52hdf.18a829d49f663934 namespace: kserve-ci-e2e-test resourceVersion: "13894" uid: ecf9d61e-a8bb-4281-90a8-6bd1380e408f reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:14:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-52hdf namespace: kserve-ci-e2e-test resourceVersion: "13425" uid: e927b850-b5b5-4cea-862e-4e4d1d93c247 kind: Event lastTimestamp: "2026-04-20T20:14:08Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.176s (2.176s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-04-20T20:14:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:14:08Z" name: isvc-xgboost-graph-predictor-669d8d6456-52hdf.18a829d521254c9d namespace: kserve-ci-e2e-test resourceVersion: "13906" uid: 2b7ee5a3-fb87-4cf1-8144-71eb91298d34 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:14:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-52hdf namespace: kserve-ci-e2e-test resourceVersion: "13425" uid: e927b850-b5b5-4cea-862e-4e4d1d93c247 kind: Event lastTimestamp: "2026-04-20T20:14:08Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-20T20:14:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:14:08Z" name: isvc-xgboost-graph-predictor-669d8d6456-52hdf.18a829d5244f4c68 namespace: kserve-ci-e2e-test resourceVersion: "13907" uid: afbf0c44-1028-43e3-8cb0-82ed0dd4841d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:14:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-52hdf namespace: kserve-ci-e2e-test resourceVersion: "13425" uid: e927b850-b5b5-4cea-862e-4e4d1d93c247 kind: Event lastTimestamp: "2026-04-20T20:14:08Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:14:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:14:08Z" name: isvc-xgboost-graph-predictor-669d8d6456-52hdf.18a829d524c602f2 namespace: kserve-ci-e2e-test resourceVersion: "13908" uid: d97244da-ac80-43c5-9266-b3d8dbd5d6a3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-20T20:14:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-52hdf namespace: kserve-ci-e2e-test resourceVersion: "13425" uid: e927b850-b5b5-4cea-862e-4e4d1d93c247 kind: Event lastTimestamp: "2026-04-20T20:14:55Z" message: 'Readiness probe failed: dial tcp 10.134.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-20T20:14:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:14:55Z" name: isvc-xgboost-graph-predictor-669d8d6456-52hdf.18a829d54d180332 namespace: kserve-ci-e2e-test resourceVersion: "14340" uid: 7d158eb6-50eb-4940-bcb6-6c18bb835f84 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-52hdf namespace: kserve-ci-e2e-test resourceVersion: "13425" uid: e927b850-b5b5-4cea-862e-4e4d1d93c247 kind: Event lastTimestamp: "2026-04-20T20:15:40Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-20T20:15:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:15:40Z" name: isvc-xgboost-graph-predictor-669d8d6456-52hdf.18a829ea9afadded namespace: kserve-ci-e2e-test resourceVersion: "15042" uid: d7a2530b-193b-4960-a2a7-74a5ef919293 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-52hdf namespace: kserve-ci-e2e-test resourceVersion: "13425" uid: e927b850-b5b5-4cea-862e-4e4d1d93c247 kind: Event lastTimestamp: "2026-04-20T20:15:40Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:15:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:15:40Z" name: isvc-xgboost-graph-predictor-669d8d6456-52hdf.18a829ea9afb3a3d namespace: kserve-ci-e2e-test resourceVersion: "15044" uid: 257240ab-fb30-4cb3-84fc-7f06d6ab7678 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:13:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-669d8d6456 namespace: kserve-ci-e2e-test resourceVersion: "13416" uid: 35217386-df4a-444c-a6a3-855b29663f37 kind: Event lastTimestamp: "2026-04-20T20:13:30Z" message: 'Created pod: isvc-xgboost-graph-predictor-669d8d6456-52hdf' metadata: creationTimestamp: "2026-04-20T20:13:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:13:30Z" name: isvc-xgboost-graph-predictor-669d8d6456.18a829cc41d3a551 namespace: kserve-ci-e2e-test resourceVersion: "13424" uid: f6c7bcbe-3a3e-4d14-bf2c-8f948709ddba reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:13:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13414" uid: b634bb0b-2d67-4bef-aeaf-37305ea333c6 kind: Event lastTimestamp: "2026-04-20T20:13:30Z" message: Scaled up replica set isvc-xgboost-graph-predictor-669d8d6456 from 0 to 1 metadata: creationTimestamp: "2026-04-20T20:13:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:13:30Z" name: isvc-xgboost-graph-predictor.18a829cc3f5ee16a namespace: kserve-ci-e2e-test resourceVersion: "13418" uid: 25b9f86b-b7fc-456c-b729-6a289fa76717 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-20T20:13:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13415" uid: a6f15121-4d96-44cc-b1b7-46fa625ef983 kind: Event lastTimestamp: "2026-04-20T20:15:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:13:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:15:30Z" name: isvc-xgboost-graph-predictor.18a829cfbde3589f namespace: kserve-ci-e2e-test resourceVersion: "14819" uid: 07f1001e-c17a-420a-8b45-991ed643ce44 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-20T20:13:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13415" uid: a6f15121-4d96-44cc-b1b7-46fa625ef983 kind: Event lastTimestamp: "2026-04-20T20:15:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:13:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:15:30Z" name: isvc-xgboost-graph-predictor.18a829cfbe54e23d namespace: kserve-ci-e2e-test resourceVersion: "14820" uid: 17925acb-90ca-4a87-be3e-dcaa812f4616 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:15:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13655" uid: a6f15121-4d96-44cc-b1b7-46fa625ef983 kind: Event lastTimestamp: "2026-04-20T20:15:15Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:15:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:15:15Z" name: isvc-xgboost-graph-predictor.18a829e138174b2e namespace: kserve-ci-e2e-test resourceVersion: "14671" uid: 162eacd6-f45d-4e9c-ac1c-7d56e3fe4675 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:15:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13655" uid: a6f15121-4d96-44cc-b1b7-46fa625ef983 kind: Event lastTimestamp: "2026-04-20T20:15:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:15:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:15:15Z" name: isvc-xgboost-graph-predictor.18a829e1388e031d namespace: kserve-ci-e2e-test resourceVersion: "14672" uid: 6f45b694-579e-46d7-9765-7c0596096938 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-20T20:13:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "13397" uid: 4d034802-29f4-441c-867c-dae6bfee2808 kind: Event lastTimestamp: "2026-04-20T20:15:05Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-20T20:13:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:15:05Z" name: isvc-xgboost-graph.18a829cc3f3ec761 namespace: kserve-ci-e2e-test resourceVersion: "14591" uid: 4e6ff9a5-0796-4ef9-b399-622b30f40163 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14589" uid: 4d034802-29f4-441c-867c-dae6bfee2808 kind: Event lastTimestamp: "2026-04-20T20:15:05Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-04-20T20:15:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:15:05Z" name: isvc-xgboost-graph.18a829e25c179301 namespace: kserve-ci-e2e-test resourceVersion: "14590" uid: e44b009d-e570-4a3f-b070-c535aab2523f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-20T20:15:29.943400Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-76487779f8-46tnx namespace: kserve-ci-e2e-test resourceVersion: "14793" uid: ddf372af-98f8-41fc-a69c-f432c159fdbe kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-76487779f8-46tnx to ip-10-0-131-234.ec2.internal metadata: creationTimestamp: "2026-04-20T20:15:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-20T20:15:29Z" name: model-chainer-76487779f8-46tnx.18a829e81ffbc56a namespace: kserve-ci-e2e-test resourceVersion: "14799" uid: 77feacec-fe07-41d7-88c3-6f29fa2e2667 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58dbb994fc-hbcnb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:30Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-76487779f8-46tnx namespace: kserve-ci-e2e-test resourceVersion: "14796" uid: ddf372af-98f8-41fc-a69c-f432c159fdbe kind: Event lastTimestamp: "2026-04-20T20:15:30Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-serving-cert" not found' metadata: creationTimestamp: "2026-04-20T20:15:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:15:30Z" name: model-chainer-76487779f8-46tnx.18a829e82ad5fb69 namespace: kserve-ci-e2e-test resourceVersion: "14811" uid: ba86ac9e-18e4-43f0-a842-246819c48c1f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:30Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-76487779f8-46tnx namespace: kserve-ci-e2e-test resourceVersion: "14802" uid: ddf372af-98f8-41fc-a69c-f432c159fdbe kind: Event lastTimestamp: "2026-04-20T20:15:30Z" message: Add eth0 [10.134.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-20T20:15:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-20T20:15:30Z" name: model-chainer-76487779f8-46tnx.18a829e85cb1589c namespace: kserve-ci-e2e-test resourceVersion: "14837" uid: 4336e6ab-5a38-4b53-a924-a40c5ce92b51 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-76487779f8-46tnx namespace: kserve-ci-e2e-test resourceVersion: "14796" uid: ddf372af-98f8-41fc-a69c-f432c159fdbe kind: Event lastTimestamp: "2026-04-20T20:15:30Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:0169f8fa51f9c0b83c5c9a121f51af9f3e3cdcc48ff1b00c2fcaa12b38a7914f" already present on machine metadata: creationTimestamp: "2026-04-20T20:15:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:15:30Z" name: model-chainer-76487779f8-46tnx.18a829e85dea5fc2 namespace: kserve-ci-e2e-test resourceVersion: "14839" uid: ee253e52-251e-4624-b9fb-f59ee7b4466d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-76487779f8-46tnx namespace: kserve-ci-e2e-test resourceVersion: "14796" uid: ddf372af-98f8-41fc-a69c-f432c159fdbe kind: Event lastTimestamp: "2026-04-20T20:15:31Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-04-20T20:15:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:15:31Z" name: model-chainer-76487779f8-46tnx.18a829e8610bd161 namespace: kserve-ci-e2e-test resourceVersion: "14840" uid: c2f22d0b-e54a-43da-b7a0-b3e79e0e1629 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-76487779f8-46tnx namespace: kserve-ci-e2e-test resourceVersion: "14796" uid: ddf372af-98f8-41fc-a69c-f432c159fdbe kind: Event lastTimestamp: "2026-04-20T20:15:31Z" message: Started container model-chainer metadata: creationTimestamp: "2026-04-20T20:15:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:15:31Z" name: model-chainer-76487779f8-46tnx.18a829e861bddd48 namespace: kserve-ci-e2e-test resourceVersion: "14841" uid: 1e748292-2604-45dc-a481-1c92ab932157 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-76487779f8-46tnx namespace: kserve-ci-e2e-test resourceVersion: "14796" uid: ddf372af-98f8-41fc-a69c-f432c159fdbe kind: Event lastTimestamp: "2026-04-20T20:15:40Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-04-20T20:15:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:15:40Z" name: model-chainer-76487779f8-46tnx.18a829ea7f9dff7c namespace: kserve-ci-e2e-test resourceVersion: "14938" uid: 9421a488-c9ee-4dfb-a18f-0f4ce0a63cdf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-20T20:15:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-76487779f8-46tnx namespace: kserve-ci-e2e-test resourceVersion: "14796" uid: ddf372af-98f8-41fc-a69c-f432c159fdbe kind: Event lastTimestamp: "2026-04-20T20:16:07Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-20T20:15:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:16:07Z" name: model-chainer-76487779f8-46tnx.18a829eb045ad46a namespace: kserve-ci-e2e-test resourceVersion: "15357" uid: 8be98d46-039b-49c8-82a8-8d3e30c391ac reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-76487779f8 namespace: kserve-ci-e2e-test resourceVersion: "14790" uid: 7e71dc75-a4e2-4a92-b0ee-7339ef5642a9 kind: Event lastTimestamp: "2026-04-20T20:15:29Z" message: 'Created pod: model-chainer-76487779f8-46tnx' metadata: creationTimestamp: "2026-04-20T20:15:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:15:29Z" name: model-chainer-76487779f8.18a829e81f894009 namespace: kserve-ci-e2e-test resourceVersion: "14795" uid: 7d8aae31-f1f3-42cb-a495-624233c91cd5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "14788" uid: 853e3b66-ad8d-479d-91cd-db0e201aa5dd kind: Event lastTimestamp: "2026-04-20T20:15:29Z" message: Scaled up replica set model-chainer-76487779f8 from 0 to 1 metadata: creationTimestamp: "2026-04-20T20:15:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:15:29Z" name: model-chainer.18a829e81eb4928d namespace: kserve-ci-e2e-test resourceVersion: "14791" uid: 06cc1e45-4ec5-443f-aa10-9bc4932011da reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:30Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "14805" uid: e01317e7-95a7-4b16-9e2e-80675df04b8d kind: Event lastTimestamp: "2026-04-20T20:15:30Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-04-20T20:15:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:15:30Z" name: model-chainer.18a829e824d07980 namespace: kserve-ci-e2e-test resourceVersion: "14807" uid: d8108a6a-0b1c-4ce7-b7a0-48bde768fae5 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-20T20:25:55.184278Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-63de8-85847f4f4d-lglk2 namespace: kserve-ci-e2e-test resourceVersion: "20096" uid: 526ea70f-30fa-4e11-9f6f-f13da9aef4e5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-63de8-85847f4f4d-lglk2 to ip-10-0-131-234.ec2.internal metadata: creationTimestamp: "2026-04-20T20:25:55Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-20T20:25:55Z" name: sequence-graph-63de8-85847f4f4d-lglk2.18a82a79b33e7ad3 namespace: kserve-ci-e2e-test resourceVersion: "20101" uid: 65229bc2-0651-4369-9096-b3253fc0504b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58dbb994fc-hbcnb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:55Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-63de8-85847f4f4d-lglk2 namespace: kserve-ci-e2e-test resourceVersion: "20098" uid: 526ea70f-30fa-4e11-9f6f-f13da9aef4e5 kind: Event lastTimestamp: "2026-04-20T20:25:55Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-63de8-serving-cert" not found' metadata: creationTimestamp: "2026-04-20T20:25:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:25:55Z" name: sequence-graph-63de8-85847f4f4d-lglk2.18a82a79c1332cac namespace: kserve-ci-e2e-test resourceVersion: "20118" uid: 5b83a8a0-046a-4a15-aa17-03e95e8616ce reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:56Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-63de8-85847f4f4d-lglk2 namespace: kserve-ci-e2e-test resourceVersion: "20106" uid: 526ea70f-30fa-4e11-9f6f-f13da9aef4e5 kind: Event lastTimestamp: "2026-04-20T20:25:56Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-20T20:25:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-20T20:25:56Z" name: sequence-graph-63de8-85847f4f4d-lglk2.18a82a79f01ebdfd namespace: kserve-ci-e2e-test resourceVersion: "20132" uid: 0bfdde6b-49e4-42a3-9b1e-a207c081b22a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-63de8} kind: Pod name: sequence-graph-63de8-85847f4f4d-lglk2 namespace: kserve-ci-e2e-test resourceVersion: "20098" uid: 526ea70f-30fa-4e11-9f6f-f13da9aef4e5 kind: Event lastTimestamp: "2026-04-20T20:25:56Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:0169f8fa51f9c0b83c5c9a121f51af9f3e3cdcc48ff1b00c2fcaa12b38a7914f" already present on machine metadata: creationTimestamp: "2026-04-20T20:25:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:25:56Z" name: sequence-graph-63de8-85847f4f4d-lglk2.18a82a79f1478f96 namespace: kserve-ci-e2e-test resourceVersion: "20134" uid: 17f63a7e-390a-4fa1-a28a-962d6a233c11 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-63de8} kind: Pod name: sequence-graph-63de8-85847f4f4d-lglk2 namespace: kserve-ci-e2e-test resourceVersion: "20098" uid: 526ea70f-30fa-4e11-9f6f-f13da9aef4e5 kind: Event lastTimestamp: "2026-04-20T20:25:56Z" message: 'Created container: sequence-graph-63de8' metadata: creationTimestamp: "2026-04-20T20:25:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:25:56Z" name: sequence-graph-63de8-85847f4f4d-lglk2.18a82a79f483ab54 namespace: kserve-ci-e2e-test resourceVersion: "20137" uid: da313103-8123-4c92-8766-c6df7810a813 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-63de8} kind: Pod name: sequence-graph-63de8-85847f4f4d-lglk2 namespace: kserve-ci-e2e-test resourceVersion: "20098" uid: 526ea70f-30fa-4e11-9f6f-f13da9aef4e5 kind: Event lastTimestamp: "2026-04-20T20:25:56Z" message: Started container sequence-graph-63de8 metadata: creationTimestamp: "2026-04-20T20:25:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:25:56Z" name: sequence-graph-63de8-85847f4f4d-lglk2.18a82a79f4faf4a8 namespace: kserve-ci-e2e-test resourceVersion: "20138" uid: a4776c37-48b3-4a0e-bc06-8b9d78d0cfa5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:26:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-63de8} kind: Pod name: sequence-graph-63de8-85847f4f4d-lglk2 namespace: kserve-ci-e2e-test resourceVersion: "20098" uid: 526ea70f-30fa-4e11-9f6f-f13da9aef4e5 kind: Event lastTimestamp: "2026-04-20T20:26:05Z" message: Stopping container sequence-graph-63de8 metadata: creationTimestamp: "2026-04-20T20:26:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:26:05Z" name: sequence-graph-63de8-85847f4f4d-lglk2.18a82a7c0cd5b764 namespace: kserve-ci-e2e-test resourceVersion: "20231" uid: 1169a9a7-863b-4bb4-9999-72dab7d029e7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-20T20:26:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-63de8} kind: Pod name: sequence-graph-63de8-85847f4f4d-lglk2 namespace: kserve-ci-e2e-test resourceVersion: "20098" uid: 526ea70f-30fa-4e11-9f6f-f13da9aef4e5 kind: Event lastTimestamp: "2026-04-20T20:26:33Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-20T20:26:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:26:33Z" name: sequence-graph-63de8-85847f4f4d-lglk2.18a82a7cbde998ad namespace: kserve-ci-e2e-test resourceVersion: "20626" uid: 5361064f-49c6-4e82-b6c9-5466714db8c5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:55Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-63de8-85847f4f4d namespace: kserve-ci-e2e-test resourceVersion: "20093" uid: 2e8d8e99-89db-43eb-b956-948cb80b2f78 kind: Event lastTimestamp: "2026-04-20T20:25:55Z" message: 'Created pod: sequence-graph-63de8-85847f4f4d-lglk2' metadata: creationTimestamp: "2026-04-20T20:25:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:25:55Z" name: sequence-graph-63de8-85847f4f4d.18a82a79b2bef633 namespace: kserve-ci-e2e-test resourceVersion: "20097" uid: 513c6d6e-7d9c-44ab-98bf-4377985587d6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:55Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-63de8 namespace: kserve-ci-e2e-test resourceVersion: "20091" uid: 38946031-3099-47fa-9ff0-0cf78ab4059e kind: Event lastTimestamp: "2026-04-20T20:25:55Z" message: Scaled up replica set sequence-graph-63de8-85847f4f4d from 0 to 1 metadata: creationTimestamp: "2026-04-20T20:25:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:25:55Z" name: sequence-graph-63de8.18a82a79b20e39a9 namespace: kserve-ci-e2e-test resourceVersion: "20094" uid: 13f2e0f8-060c-462b-819a-406b5dc63e71 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:55Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-63de8 namespace: kserve-ci-e2e-test resourceVersion: "20109" uid: d3b6988f-85e4-4c5a-9795-129347d7bc0e kind: Event lastTimestamp: "2026-04-20T20:25:55Z" message: InferenceGraph [sequence-graph-63de8] is Ready metadata: creationTimestamp: "2026-04-20T20:25:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:25:55Z" name: sequence-graph-63de8.18a82a79b6c819ac namespace: kserve-ci-e2e-test resourceVersion: "20111" uid: 9ee61483-eca1-4bde-992d-58af917852d5 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-20T20:16:40.332868Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-e6f34-55f69b6976-9c549 namespace: kserve-ci-e2e-test resourceVersion: "15646" uid: 9ab0dd56-0bd3-4617-952f-bcd860eceee1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-e6f34-55f69b6976-9c549 to ip-10-0-131-234.ec2.internal metadata: creationTimestamp: "2026-04-20T20:16:40Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-20T20:16:40Z" name: sequence-graph-e6f34-55f69b6976-9c549.18a829f88385cd6a namespace: kserve-ci-e2e-test resourceVersion: "15651" uid: 9391f909-4221-4720-810f-4249601aeb69 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58dbb994fc-hbcnb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:16:40Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-e6f34-55f69b6976-9c549 namespace: kserve-ci-e2e-test resourceVersion: "15654" uid: 9ab0dd56-0bd3-4617-952f-bcd860eceee1 kind: Event lastTimestamp: "2026-04-20T20:16:40Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-20T20:16:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-20T20:16:40Z" name: sequence-graph-e6f34-55f69b6976-9c549.18a829f89c82bb6f namespace: kserve-ci-e2e-test resourceVersion: "15674" uid: 2f7c4365-f192-45a6-81a2-65bb867f405e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:16:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-e6f34} kind: Pod name: sequence-graph-e6f34-55f69b6976-9c549 namespace: kserve-ci-e2e-test resourceVersion: "15649" uid: 9ab0dd56-0bd3-4617-952f-bcd860eceee1 kind: Event lastTimestamp: "2026-04-20T20:16:40Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:0169f8fa51f9c0b83c5c9a121f51af9f3e3cdcc48ff1b00c2fcaa12b38a7914f" already present on machine metadata: creationTimestamp: "2026-04-20T20:16:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:16:40Z" name: sequence-graph-e6f34-55f69b6976-9c549.18a829f89db25064 namespace: kserve-ci-e2e-test resourceVersion: "15676" uid: c06b0820-7f2d-4ca7-9857-ed7b6509d0b3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:16:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-e6f34} kind: Pod name: sequence-graph-e6f34-55f69b6976-9c549 namespace: kserve-ci-e2e-test resourceVersion: "15649" uid: 9ab0dd56-0bd3-4617-952f-bcd860eceee1 kind: Event lastTimestamp: "2026-04-20T20:16:40Z" message: 'Created container: sequence-graph-e6f34' metadata: creationTimestamp: "2026-04-20T20:16:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:16:40Z" name: sequence-graph-e6f34-55f69b6976-9c549.18a829f8a0baa0a5 namespace: kserve-ci-e2e-test resourceVersion: "15677" uid: 675e1b4f-6e3f-4d7c-aa5e-524a312613fe reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:16:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-e6f34} kind: Pod name: sequence-graph-e6f34-55f69b6976-9c549 namespace: kserve-ci-e2e-test resourceVersion: "15649" uid: 9ab0dd56-0bd3-4617-952f-bcd860eceee1 kind: Event lastTimestamp: "2026-04-20T20:16:40Z" message: Started container sequence-graph-e6f34 metadata: creationTimestamp: "2026-04-20T20:16:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:16:40Z" name: sequence-graph-e6f34-55f69b6976-9c549.18a829f8a13189d9 namespace: kserve-ci-e2e-test resourceVersion: "15678" uid: da67c578-b40e-4e7b-9a10-4609732e9fcb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:24:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-e6f34} kind: Pod name: sequence-graph-e6f34-55f69b6976-9c549 namespace: kserve-ci-e2e-test resourceVersion: "15649" uid: 9ab0dd56-0bd3-4617-952f-bcd860eceee1 kind: Event lastTimestamp: "2026-04-20T20:24:54Z" message: Stopping container sequence-graph-e6f34 metadata: creationTimestamp: "2026-04-20T20:24:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:24:54Z" name: sequence-graph-e6f34-55f69b6976-9c549.18a82a6baf6b8b7c namespace: kserve-ci-e2e-test resourceVersion: "19258" uid: ac88569b-ce94-4088-becc-354a4500b8e2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-20T20:24:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-e6f34} kind: Pod name: sequence-graph-e6f34-55f69b6976-9c549 namespace: kserve-ci-e2e-test resourceVersion: "15649" uid: 9ab0dd56-0bd3-4617-952f-bcd860eceee1 kind: Event lastTimestamp: "2026-04-20T20:25:22Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-20T20:24:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:25:22Z" name: sequence-graph-e6f34-55f69b6976-9c549.18a82a6c4a1d6c4c namespace: kserve-ci-e2e-test resourceVersion: "19661" uid: daa762bf-2b6d-445b-bf89-943bc164b58e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:16:40Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-e6f34-55f69b6976 namespace: kserve-ci-e2e-test resourceVersion: "15643" uid: 90bdb76a-66c5-47ba-8705-2b8d5adde47c kind: Event lastTimestamp: "2026-04-20T20:16:40Z" message: 'Created pod: sequence-graph-e6f34-55f69b6976-9c549' metadata: creationTimestamp: "2026-04-20T20:16:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:16:40Z" name: sequence-graph-e6f34-55f69b6976.18a829f8831d73fe namespace: kserve-ci-e2e-test resourceVersion: "15648" uid: bd5fea9a-9240-41c1-ad44-1990694950e5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:16:40Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-e6f34 namespace: kserve-ci-e2e-test resourceVersion: "15641" uid: 42a9764d-49c4-456c-9c6b-591a090a6057 kind: Event lastTimestamp: "2026-04-20T20:16:40Z" message: Scaled up replica set sequence-graph-e6f34-55f69b6976 from 0 to 1 metadata: creationTimestamp: "2026-04-20T20:16:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:16:40Z" name: sequence-graph-e6f34.18a829f8822ff52b namespace: kserve-ci-e2e-test resourceVersion: "15644" uid: 92bb2c13-201f-489b-954d-3fb95cb40dce reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:16:40Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-e6f34 namespace: kserve-ci-e2e-test resourceVersion: "15657" uid: e5174afb-14c1-4e7b-a785-a00b2f531622 kind: Event lastTimestamp: "2026-04-20T20:16:40Z" message: InferenceGraph [sequence-graph-e6f34] is Ready metadata: creationTimestamp: "2026-04-20T20:16:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:16:40Z" name: sequence-graph-e6f34.18a829f886aa7bd9 namespace: kserve-ci-e2e-test resourceVersion: "15659" uid: c06928fa-3476-48e7-b318-85ae4e4a6016 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:16:40Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-e6f34 namespace: kserve-ci-e2e-test resourceVersion: "15633" uid: e5174afb-14c1-4e7b-a785-a00b2f531622 kind: Event lastTimestamp: "2026-04-20T20:16:40Z" message: 'Failed to update status for InferenceGraph "sequence-graph-e6f34": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-e6f34": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-20T20:16:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:16:40Z" name: sequence-graph-e6f34.18a829f888d6032e namespace: kserve-ci-e2e-test resourceVersion: "15661" uid: 51e5ee0c-8d84-4e3e-811f-af39e903a717 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:16:40Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-e6f34 namespace: kserve-ci-e2e-test resourceVersion: "15633" uid: e5174afb-14c1-4e7b-a785-a00b2f531622 kind: Event lastTimestamp: "2026-04-20T20:16:40Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-e6f34": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-20T20:16:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:16:40Z" name: sequence-graph-e6f34.18a829f888d6544d namespace: kserve-ci-e2e-test resourceVersion: "15664" uid: 5a465736-234d-4d00-a107-ec20568e25fe reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:16:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-e6f34 namespace: kserve-ci-e2e-test resourceVersion: "15642" uid: ef13874e-c8ad-4f48-83d1-418234a78b78 kind: Event lastTimestamp: "2026-04-20T20:16:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:16:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:16:55Z" name: sequence-graph-e6f34.18a829fc00a84821 namespace: kserve-ci-e2e-test resourceVersion: "15780" uid: 43c2ecd5-9c66-420a-ac30-2e037c2c8dae reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:16:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-e6f34 namespace: kserve-ci-e2e-test resourceVersion: "15642" uid: ef13874e-c8ad-4f48-83d1-418234a78b78 kind: Event lastTimestamp: "2026-04-20T20:16:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:16:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:16:55Z" name: sequence-graph-e6f34.18a829fc01057e76 namespace: kserve-ci-e2e-test resourceVersion: "15782" uid: 955b1271-7321-4124-a560-6c081a203993 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:17:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-e6f34 namespace: kserve-ci-e2e-test resourceVersion: "15781" uid: ef13874e-c8ad-4f48-83d1-418234a78b78 kind: Event lastTimestamp: "2026-04-20T20:17:10Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:17:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:17:10Z" name: sequence-graph-e6f34.18a829ff7fb013e8 namespace: kserve-ci-e2e-test resourceVersion: "15890" uid: 60c078c1-3098-4350-94b2-29b3cef4885e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:17:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-e6f34 namespace: kserve-ci-e2e-test resourceVersion: "15781" uid: ef13874e-c8ad-4f48-83d1-418234a78b78 kind: Event lastTimestamp: "2026-04-20T20:17:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:17:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:17:10Z" name: sequence-graph-e6f34.18a829ff800cfef7 namespace: kserve-ci-e2e-test resourceVersion: "15892" uid: cb3d4f96-27b3-4bf4-97b9-026a91104b76 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-20T20:27:05.489347Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-fe410-7759986577-6jbfq namespace: kserve-ci-e2e-test resourceVersion: "20917" uid: a518fbed-cd34-47b8-8cfd-82217f9c49e2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-fe410-7759986577-6jbfq to ip-10-0-131-234.ec2.internal metadata: creationTimestamp: "2026-04-20T20:27:05Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-20T20:27:05Z" name: sequence-graph-fe410-7759986577-6jbfq.18a82a8a11c0a17a namespace: kserve-ci-e2e-test resourceVersion: "20922" uid: 69cd809f-140c-427d-84b5-79541cfaacbb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58dbb994fc-hbcnb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:27:05Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-fe410-7759986577-6jbfq namespace: kserve-ci-e2e-test resourceVersion: "20926" uid: a518fbed-cd34-47b8-8cfd-82217f9c49e2 kind: Event lastTimestamp: "2026-04-20T20:27:05Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-20T20:27:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-20T20:27:05Z" name: sequence-graph-fe410-7759986577-6jbfq.18a82a8a2ab2ce54 namespace: kserve-ci-e2e-test resourceVersion: "20937" uid: 82f475f1-1ff7-42cd-a453-b064d55b92da reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:27:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-fe410} kind: Pod name: sequence-graph-fe410-7759986577-6jbfq namespace: kserve-ci-e2e-test resourceVersion: "20921" uid: a518fbed-cd34-47b8-8cfd-82217f9c49e2 kind: Event lastTimestamp: "2026-04-20T20:27:05Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:0169f8fa51f9c0b83c5c9a121f51af9f3e3cdcc48ff1b00c2fcaa12b38a7914f" already present on machine metadata: creationTimestamp: "2026-04-20T20:27:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:27:05Z" name: sequence-graph-fe410-7759986577-6jbfq.18a82a8a2bdda010 namespace: kserve-ci-e2e-test resourceVersion: "20939" uid: c208f37b-daef-4379-bb21-e746f1bffc3b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:27:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-fe410} kind: Pod name: sequence-graph-fe410-7759986577-6jbfq namespace: kserve-ci-e2e-test resourceVersion: "20921" uid: a518fbed-cd34-47b8-8cfd-82217f9c49e2 kind: Event lastTimestamp: "2026-04-20T20:27:05Z" message: 'Created container: sequence-graph-fe410' metadata: creationTimestamp: "2026-04-20T20:27:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:27:05Z" name: sequence-graph-fe410-7759986577-6jbfq.18a82a8a2edd3a2e namespace: kserve-ci-e2e-test resourceVersion: "20940" uid: 77a88674-3bcb-4f25-bc26-fa0c101feb06 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:27:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-fe410} kind: Pod name: sequence-graph-fe410-7759986577-6jbfq namespace: kserve-ci-e2e-test resourceVersion: "20921" uid: a518fbed-cd34-47b8-8cfd-82217f9c49e2 kind: Event lastTimestamp: "2026-04-20T20:27:05Z" message: Started container sequence-graph-fe410 metadata: creationTimestamp: "2026-04-20T20:27:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:27:05Z" name: sequence-graph-fe410-7759986577-6jbfq.18a82a8a2f557f05 namespace: kserve-ci-e2e-test resourceVersion: "20941" uid: 4cca4d5d-e74c-4624-823f-9ff6f6be1360 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-fe410} kind: Pod name: sequence-graph-fe410-7759986577-6jbfq namespace: kserve-ci-e2e-test resourceVersion: "20921" uid: a518fbed-cd34-47b8-8cfd-82217f9c49e2 kind: Event lastTimestamp: "2026-04-20T20:35:20Z" message: Stopping container sequence-graph-fe410 metadata: creationTimestamp: "2026-04-20T20:35:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:35:20Z" name: sequence-graph-fe410-7759986577-6jbfq.18a82afd406b72a6 namespace: kserve-ci-e2e-test resourceVersion: "24509" uid: c0ab954e-35b0-48e0-8247-0a536b0c9d61 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-20T20:35:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-fe410} kind: Pod name: sequence-graph-fe410-7759986577-6jbfq namespace: kserve-ci-e2e-test resourceVersion: "20921" uid: a518fbed-cd34-47b8-8cfd-82217f9c49e2 kind: Event lastTimestamp: "2026-04-20T20:35:47Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-20T20:35:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:35:47Z" name: sequence-graph-fe410-7759986577-6jbfq.18a82afdc7763b2a namespace: kserve-ci-e2e-test resourceVersion: "24906" uid: 0809ef13-d3e8-4922-bcb8-a4b718e6bfee reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:27:05Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-fe410-7759986577 namespace: kserve-ci-e2e-test resourceVersion: "20913" uid: f6c62538-4a53-4233-89e4-3e82f1addb07 kind: Event lastTimestamp: "2026-04-20T20:27:05Z" message: 'Created pod: sequence-graph-fe410-7759986577-6jbfq' metadata: creationTimestamp: "2026-04-20T20:27:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:27:05Z" name: sequence-graph-fe410-7759986577.18a82a8a111e7290 namespace: kserve-ci-e2e-test resourceVersion: "20918" uid: 8140fd2c-3562-4234-882d-7603e4b0bee7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:27:05Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-fe410 namespace: kserve-ci-e2e-test resourceVersion: "20911" uid: d7f0e921-9c1e-4ad5-8f10-135ae3b97adb kind: Event lastTimestamp: "2026-04-20T20:27:05Z" message: Scaled up replica set sequence-graph-fe410-7759986577 from 0 to 1 metadata: creationTimestamp: "2026-04-20T20:27:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:27:05Z" name: sequence-graph-fe410.18a82a8a10539c6d namespace: kserve-ci-e2e-test resourceVersion: "20915" uid: 2616bd3e-b6c4-4f23-b17a-591cb7a02260 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:27:05Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-fe410 namespace: kserve-ci-e2e-test resourceVersion: "20930" uid: 21562c09-deac-4324-84c1-900814a4fa0f kind: Event lastTimestamp: "2026-04-20T20:27:05Z" message: InferenceGraph [sequence-graph-fe410] is Ready metadata: creationTimestamp: "2026-04-20T20:27:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:27:05Z" name: sequence-graph-fe410.18a82a8a15c7719f namespace: kserve-ci-e2e-test resourceVersion: "20932" uid: a5ff41ea-3f83-438e-a412-ae96f81cf218 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:27:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-fe410 namespace: kserve-ci-e2e-test resourceVersion: "20912" uid: 43be5b2c-59b4-4931-b02c-2fe5f263407b kind: Event lastTimestamp: "2026-04-20T20:27:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:27:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:27:20Z" name: sequence-graph-fe410.18a82a8d8ebe5de2 namespace: kserve-ci-e2e-test resourceVersion: "21048" uid: 988bad6d-77a7-473f-b889-64f2681ac0a4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:27:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-fe410 namespace: kserve-ci-e2e-test resourceVersion: "20912" uid: 43be5b2c-59b4-4931-b02c-2fe5f263407b kind: Event lastTimestamp: "2026-04-20T20:27:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:27:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:27:20Z" name: sequence-graph-fe410.18a82a8d8f1c2f1b namespace: kserve-ci-e2e-test resourceVersion: "21050" uid: b8529502-ae97-4d4e-9e71-d3df0eb6bd00 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:27:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-fe410 namespace: kserve-ci-e2e-test resourceVersion: "21049" uid: 43be5b2c-59b4-4931-b02c-2fe5f263407b kind: Event lastTimestamp: "2026-04-20T20:27:35Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:27:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:27:35Z" name: sequence-graph-fe410.18a82a910dc3de42 namespace: kserve-ci-e2e-test resourceVersion: "21158" uid: 10a93891-d1b0-4991-8bc5-432ae2bbf60f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:27:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-fe410 namespace: kserve-ci-e2e-test resourceVersion: "21049" uid: 43be5b2c-59b4-4931-b02c-2fe5f263407b kind: Event lastTimestamp: "2026-04-20T20:27:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:27:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:27:35Z" name: sequence-graph-fe410.18a82a910e22e8e6 namespace: kserve-ci-e2e-test resourceVersion: "21160" uid: 36d43f28-12f1-4c4f-b508-ed67b6ccc2ff reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-20T20:36:50.898043Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-57557-86766cd968-2dzqj namespace: kserve-ci-e2e-test resourceVersion: "25644" uid: 324cf3c9-ff6e-4742-bfe0-eb1d008344c2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-57557-86766cd968-2dzqj to ip-10-0-131-234.ec2.internal metadata: creationTimestamp: "2026-04-20T20:36:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-20T20:36:50Z" name: splitter-graph-57557-86766cd968-2dzqj.18a82b125ed47d15 namespace: kserve-ci-e2e-test resourceVersion: "25649" uid: 853c48e6-c148-48ed-831a-004118495d4a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58dbb994fc-hbcnb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:36:51Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-57557-86766cd968-2dzqj namespace: kserve-ci-e2e-test resourceVersion: "25647" uid: 324cf3c9-ff6e-4742-bfe0-eb1d008344c2 kind: Event lastTimestamp: "2026-04-20T20:36:51Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-57557-serving-cert" not found' metadata: creationTimestamp: "2026-04-20T20:36:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:36:51Z" name: splitter-graph-57557-86766cd968-2dzqj.18a82b126c8d725e namespace: kserve-ci-e2e-test resourceVersion: "25663" uid: b89a27e8-8d14-4948-94e6-1180d7c60520 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:36:51Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-57557-86766cd968-2dzqj namespace: kserve-ci-e2e-test resourceVersion: "25652" uid: 324cf3c9-ff6e-4742-bfe0-eb1d008344c2 kind: Event lastTimestamp: "2026-04-20T20:36:51Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-20T20:36:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-20T20:36:51Z" name: splitter-graph-57557-86766cd968-2dzqj.18a82b129bc0d54d namespace: kserve-ci-e2e-test resourceVersion: "25668" uid: ee09a86d-8cb2-49f1-aea9-6165d7384c4b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:36:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-57557} kind: Pod name: splitter-graph-57557-86766cd968-2dzqj namespace: kserve-ci-e2e-test resourceVersion: "25647" uid: 324cf3c9-ff6e-4742-bfe0-eb1d008344c2 kind: Event lastTimestamp: "2026-04-20T20:36:51Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:0169f8fa51f9c0b83c5c9a121f51af9f3e3cdcc48ff1b00c2fcaa12b38a7914f" already present on machine metadata: creationTimestamp: "2026-04-20T20:36:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:36:51Z" name: splitter-graph-57557-86766cd968-2dzqj.18a82b129cf7e373 namespace: kserve-ci-e2e-test resourceVersion: "25670" uid: 5dcc6ef9-b563-4a08-a0b4-51da721dbadf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:36:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-57557} kind: Pod name: splitter-graph-57557-86766cd968-2dzqj namespace: kserve-ci-e2e-test resourceVersion: "25647" uid: 324cf3c9-ff6e-4742-bfe0-eb1d008344c2 kind: Event lastTimestamp: "2026-04-20T20:36:51Z" message: 'Created container: splitter-graph-57557' metadata: creationTimestamp: "2026-04-20T20:36:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:36:51Z" name: splitter-graph-57557-86766cd968-2dzqj.18a82b129ff2943f namespace: kserve-ci-e2e-test resourceVersion: "25672" uid: 21b89043-f278-44bf-928f-9bb962e509c5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:36:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-57557} kind: Pod name: splitter-graph-57557-86766cd968-2dzqj namespace: kserve-ci-e2e-test resourceVersion: "25647" uid: 324cf3c9-ff6e-4742-bfe0-eb1d008344c2 kind: Event lastTimestamp: "2026-04-20T20:36:51Z" message: Started container splitter-graph-57557 metadata: creationTimestamp: "2026-04-20T20:36:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:36:51Z" name: splitter-graph-57557-86766cd968-2dzqj.18a82b12a0695a17 namespace: kserve-ci-e2e-test resourceVersion: "25673" uid: 106237bd-d046-4b70-9aeb-d12746464c2a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:45:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-57557} kind: Pod name: splitter-graph-57557-86766cd968-2dzqj namespace: kserve-ci-e2e-test resourceVersion: "25647" uid: 324cf3c9-ff6e-4742-bfe0-eb1d008344c2 kind: Event lastTimestamp: "2026-04-20T20:45:05Z" message: Stopping container splitter-graph-57557 metadata: creationTimestamp: "2026-04-20T20:45:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:45:05Z" name: splitter-graph-57557-86766cd968-2dzqj.18a82b8585d4ca98 namespace: kserve-ci-e2e-test resourceVersion: "29414" uid: 5df778cf-614c-4e49-a700-b6883c68f1e0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-20T20:45:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-57557} kind: Pod name: splitter-graph-57557-86766cd968-2dzqj namespace: kserve-ci-e2e-test resourceVersion: "25647" uid: 324cf3c9-ff6e-4742-bfe0-eb1d008344c2 kind: Event lastTimestamp: "2026-04-20T20:45:33Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-20T20:45:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:45:33Z" name: splitter-graph-57557-86766cd968-2dzqj.18a82b8626c4d99f namespace: kserve-ci-e2e-test resourceVersion: "29646" uid: c33c1119-49b7-4293-9b98-87b4345bf894 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:36:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-57557-86766cd968 namespace: kserve-ci-e2e-test resourceVersion: "25641" uid: 26a77d5c-e156-4a23-bdbf-058e95d61b00 kind: Event lastTimestamp: "2026-04-20T20:36:50Z" message: 'Created pod: splitter-graph-57557-86766cd968-2dzqj' metadata: creationTimestamp: "2026-04-20T20:36:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:36:50Z" name: splitter-graph-57557-86766cd968.18a82b125e70858f namespace: kserve-ci-e2e-test resourceVersion: "25645" uid: 3fd25a91-4f4c-4301-9244-98814202d8ba reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:36:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-57557 namespace: kserve-ci-e2e-test resourceVersion: "25639" uid: 1af67f56-c1f8-48a4-8f2a-7cca3c25ed7a kind: Event lastTimestamp: "2026-04-20T20:36:50Z" message: Scaled up replica set splitter-graph-57557-86766cd968 from 0 to 1 metadata: creationTimestamp: "2026-04-20T20:36:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:36:50Z" name: splitter-graph-57557.18a82b125dbd3088 namespace: kserve-ci-e2e-test resourceVersion: "25642" uid: 29396b7e-2f5f-4f5f-9707-22ab25cfa3ff reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:36:50Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-57557 namespace: kserve-ci-e2e-test resourceVersion: "25657" uid: 773be7a1-b2ae-4fa9-b46d-c13ac927efa4 kind: Event lastTimestamp: "2026-04-20T20:36:50Z" message: InferenceGraph [splitter-graph-57557] is Ready metadata: creationTimestamp: "2026-04-20T20:36:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:36:50Z" name: splitter-graph-57557.18a82b1261a86b8b namespace: kserve-ci-e2e-test resourceVersion: "25659" uid: ea077320-6ce0-466f-860c-6c25b1d25a1c reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:37:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-57557 namespace: kserve-ci-e2e-test resourceVersion: "25640" uid: aae59d3e-bc9c-47bc-9297-30f3caa2ab24 kind: Event lastTimestamp: "2026-04-20T20:37:05Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:37:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:37:05Z" name: splitter-graph-57557.18a82b15dc3535f3 namespace: kserve-ci-e2e-test resourceVersion: "25782" uid: cdcffe0a-6183-45db-92ef-d5112f899c3f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:37:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-57557 namespace: kserve-ci-e2e-test resourceVersion: "25640" uid: aae59d3e-bc9c-47bc-9297-30f3caa2ab24 kind: Event lastTimestamp: "2026-04-20T20:37:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:37:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:37:05Z" name: splitter-graph-57557.18a82b15dc96b694 namespace: kserve-ci-e2e-test resourceVersion: "25784" uid: d4bcf5d3-7677-4136-ada3-889879d70039 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-20T20:35:40.596202Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-ab6b8-54bf769bd5-jnmp5 namespace: kserve-ci-e2e-test resourceVersion: "24830" uid: e972aafe-9982-429a-8ec8-39b81b77fc55 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-ab6b8-54bf769bd5-jnmp5 to ip-10-0-131-234.ec2.internal metadata: creationTimestamp: "2026-04-20T20:35:40Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-20T20:35:40Z" name: splitter-graph-ab6b8-54bf769bd5-jnmp5.18a82b0200837c55 namespace: kserve-ci-e2e-test resourceVersion: "24835" uid: d8a8d999-be66-4cd4-ad27-ccce5d079e10 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58dbb994fc-hbcnb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:40Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-ab6b8-54bf769bd5-jnmp5 namespace: kserve-ci-e2e-test resourceVersion: "24832" uid: e972aafe-9982-429a-8ec8-39b81b77fc55 kind: Event lastTimestamp: "2026-04-20T20:35:40Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-ab6b8-serving-cert" not found' metadata: creationTimestamp: "2026-04-20T20:35:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:35:40Z" name: splitter-graph-ab6b8-54bf769bd5-jnmp5.18a82b020e9f6f90 namespace: kserve-ci-e2e-test resourceVersion: "24848" uid: ce122a16-9630-4e70-a53c-0b97a69aa5db reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:41Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-ab6b8-54bf769bd5-jnmp5 namespace: kserve-ci-e2e-test resourceVersion: "24839" uid: e972aafe-9982-429a-8ec8-39b81b77fc55 kind: Event lastTimestamp: "2026-04-20T20:35:41Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-20T20:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-20T20:35:41Z" name: splitter-graph-ab6b8-54bf769bd5-jnmp5.18a82b023d698d94 namespace: kserve-ci-e2e-test resourceVersion: "24857" uid: 7e55efc2-41a8-483d-a16f-90ca0b3c615b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-ab6b8} kind: Pod name: splitter-graph-ab6b8-54bf769bd5-jnmp5 namespace: kserve-ci-e2e-test resourceVersion: "24832" uid: e972aafe-9982-429a-8ec8-39b81b77fc55 kind: Event lastTimestamp: "2026-04-20T20:35:41Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:0169f8fa51f9c0b83c5c9a121f51af9f3e3cdcc48ff1b00c2fcaa12b38a7914f" already present on machine metadata: creationTimestamp: "2026-04-20T20:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:35:41Z" name: splitter-graph-ab6b8-54bf769bd5-jnmp5.18a82b023eab3cf4 namespace: kserve-ci-e2e-test resourceVersion: "24860" uid: b7091862-d6c5-4e18-98cd-ea3d8813ff54 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-ab6b8} kind: Pod name: splitter-graph-ab6b8-54bf769bd5-jnmp5 namespace: kserve-ci-e2e-test resourceVersion: "24832" uid: e972aafe-9982-429a-8ec8-39b81b77fc55 kind: Event lastTimestamp: "2026-04-20T20:35:41Z" message: 'Created container: splitter-graph-ab6b8' metadata: creationTimestamp: "2026-04-20T20:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:35:41Z" name: splitter-graph-ab6b8-54bf769bd5-jnmp5.18a82b0241c2eef0 namespace: kserve-ci-e2e-test resourceVersion: "24862" uid: 7cd35bd1-864b-4702-94e8-0aa8795537f0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-ab6b8} kind: Pod name: splitter-graph-ab6b8-54bf769bd5-jnmp5 namespace: kserve-ci-e2e-test resourceVersion: "24832" uid: e972aafe-9982-429a-8ec8-39b81b77fc55 kind: Event lastTimestamp: "2026-04-20T20:35:41Z" message: Started container splitter-graph-ab6b8 metadata: creationTimestamp: "2026-04-20T20:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:35:41Z" name: splitter-graph-ab6b8-54bf769bd5-jnmp5.18a82b02423b901f namespace: kserve-ci-e2e-test resourceVersion: "24863" uid: 080582bb-9e3d-4296-9983-0f6b6b4f22f8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-ab6b8} kind: Pod name: splitter-graph-ab6b8-54bf769bd5-jnmp5 namespace: kserve-ci-e2e-test resourceVersion: "24832" uid: e972aafe-9982-429a-8ec8-39b81b77fc55 kind: Event lastTimestamp: "2026-04-20T20:35:50Z" message: Stopping container splitter-graph-ab6b8 metadata: creationTimestamp: "2026-04-20T20:35:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:35:50Z" name: splitter-graph-ab6b8-54bf769bd5-jnmp5.18a82b045991e178 namespace: kserve-ci-e2e-test resourceVersion: "24956" uid: d99095a6-fefd-4626-9a24-9754dc707d22 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-20T20:35:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-ab6b8} kind: Pod name: splitter-graph-ab6b8-54bf769bd5-jnmp5 namespace: kserve-ci-e2e-test resourceVersion: "24832" uid: e972aafe-9982-429a-8ec8-39b81b77fc55 kind: Event lastTimestamp: "2026-04-20T20:36:17Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-20T20:35:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:36:17Z" name: splitter-graph-ab6b8-54bf769bd5-jnmp5.18a82b04e17fb720 namespace: kserve-ci-e2e-test resourceVersion: "25312" uid: 45419f12-d439-460c-a84f-b2e7284b27dc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:40Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-ab6b8-54bf769bd5 namespace: kserve-ci-e2e-test resourceVersion: "24827" uid: 4cf8e087-afa3-41ba-abc5-2ec0dc155376 kind: Event lastTimestamp: "2026-04-20T20:35:40Z" message: 'Created pod: splitter-graph-ab6b8-54bf769bd5-jnmp5' metadata: creationTimestamp: "2026-04-20T20:35:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:35:40Z" name: splitter-graph-ab6b8-54bf769bd5.18a82b020026411d namespace: kserve-ci-e2e-test resourceVersion: "24831" uid: 609d4279-e7f8-4ac9-8b36-caef4257ff94 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:40Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-ab6b8 namespace: kserve-ci-e2e-test resourceVersion: "24825" uid: aad7883e-0a83-4d67-8347-5092661ee7b6 kind: Event lastTimestamp: "2026-04-20T20:35:40Z" message: Scaled up replica set splitter-graph-ab6b8-54bf769bd5 from 0 to 1 metadata: creationTimestamp: "2026-04-20T20:35:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:35:40Z" name: splitter-graph-ab6b8.18a82b01ff692589 namespace: kserve-ci-e2e-test resourceVersion: "24828" uid: 7ed39d82-8942-478b-a37a-15df9bdd7f80 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:40Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-ab6b8 namespace: kserve-ci-e2e-test resourceVersion: "24843" uid: 5bb78d54-4b2d-4dda-aa1a-ed1333629294 kind: Event lastTimestamp: "2026-04-20T20:35:40Z" message: InferenceGraph [splitter-graph-ab6b8] is Ready metadata: creationTimestamp: "2026-04-20T20:35:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:35:40Z" name: splitter-graph-ab6b8.18a82b020469f400 namespace: kserve-ci-e2e-test resourceVersion: "24846" uid: 9bcce363-0d18-4016-85c0-c213fef26255 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-20T20:13:30.152749Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-127f1-predictor-68b975dccb-dcphk namespace: kserve-ci-e2e-test resourceVersion: "13396" uid: e6cb64b5-1bab-48c6-997f-4dddb3884c8d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-127f1-predictor-68b975dccb-dcphk to ip-10-0-140-19.ec2.internal metadata: creationTimestamp: "2026-04-20T20:13:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-20T20:13:30Z" name: success-200-isvc-127f1-predictor-68b975dccb-dcphk.18a829cc3be78e83 namespace: kserve-ci-e2e-test resourceVersion: "13401" uid: 98a32cc8-f4a4-45f3-b7dc-b205ca702d69 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58dbb994fc-hbcnb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:13:30Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-127f1-predictor-68b975dccb-dcphk namespace: kserve-ci-e2e-test resourceVersion: "13399" uid: e6cb64b5-1bab-48c6-997f-4dddb3884c8d kind: Event lastTimestamp: "2026-04-20T20:13:30Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-127f1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-20T20:13:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:13:30Z" name: success-200-isvc-127f1-predictor-68b975dccb-dcphk.18a829cc47c4ecb7 namespace: kserve-ci-e2e-test resourceVersion: "13446" uid: 31f25a92-f337-47e7-89d8-b619fc268c2e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:13:31Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-127f1-predictor-68b975dccb-dcphk namespace: kserve-ci-e2e-test resourceVersion: "13410" uid: e6cb64b5-1bab-48c6-997f-4dddb3884c8d kind: Event lastTimestamp: "2026-04-20T20:13:31Z" message: Add eth0 [10.132.0.18/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-20T20:13:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-20T20:13:31Z" name: success-200-isvc-127f1-predictor-68b975dccb-dcphk.18a829cc79b7210c namespace: kserve-ci-e2e-test resourceVersion: "13500" uid: 49ae13f2-7b0c-4fc9-8061-5d5e28172a85 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:13:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-127f1-predictor-68b975dccb-dcphk namespace: kserve-ci-e2e-test resourceVersion: "13399" uid: e6cb64b5-1bab-48c6-997f-4dddb3884c8d kind: Event lastTimestamp: "2026-04-20T20:13:31Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1293" metadata: creationTimestamp: "2026-04-20T20:13:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:13:31Z" name: success-200-isvc-127f1-predictor-68b975dccb-dcphk.18a829cc7b1ec1c0 namespace: kserve-ci-e2e-test resourceVersion: "13502" uid: 66360cc4-ea8f-4290-9ee7-e459ed78a084 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:13:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-127f1-predictor-68b975dccb-dcphk namespace: kserve-ci-e2e-test resourceVersion: "13399" uid: e6cb64b5-1bab-48c6-997f-4dddb3884c8d kind: Event lastTimestamp: "2026-04-20T20:13:44Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1293" in 13.776s (13.776s including waiting). Image size: 1332215401 bytes.' metadata: creationTimestamp: "2026-04-20T20:13:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:13:44Z" name: success-200-isvc-127f1-predictor-68b975dccb-dcphk.18a829cfb04b1a10 namespace: kserve-ci-e2e-test resourceVersion: "13643" uid: 79578ead-0b8d-49dc-96e5-3190cf6330bd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:13:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-127f1-predictor-68b975dccb-dcphk namespace: kserve-ci-e2e-test resourceVersion: "13399" uid: e6cb64b5-1bab-48c6-997f-4dddb3884c8d kind: Event lastTimestamp: "2026-04-20T20:13:45Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-20T20:13:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:13:45Z" name: success-200-isvc-127f1-predictor-68b975dccb-dcphk.18a829cfb37f0339 namespace: kserve-ci-e2e-test resourceVersion: "13645" uid: 2404502f-9959-47f0-8626-4db798abd941 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:13:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-127f1-predictor-68b975dccb-dcphk namespace: kserve-ci-e2e-test resourceVersion: "13399" uid: e6cb64b5-1bab-48c6-997f-4dddb3884c8d kind: Event lastTimestamp: "2026-04-20T20:13:45Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-20T20:13:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:13:45Z" name: success-200-isvc-127f1-predictor-68b975dccb-dcphk.18a829cfb402571c namespace: kserve-ci-e2e-test resourceVersion: "13646" uid: 72327902-e98d-422e-81dc-4ef3d83205af reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:13:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-127f1-predictor-68b975dccb-dcphk namespace: kserve-ci-e2e-test resourceVersion: "13399" uid: e6cb64b5-1bab-48c6-997f-4dddb3884c8d kind: Event lastTimestamp: "2026-04-20T20:13:45Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-04-20T20:13:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:13:45Z" name: success-200-isvc-127f1-predictor-68b975dccb-dcphk.18a829cfb411679d namespace: kserve-ci-e2e-test resourceVersion: "13649" uid: ae3fad1b-eedd-4b45-a233-cbd0f90285c1 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:13:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-127f1-predictor-68b975dccb-dcphk namespace: kserve-ci-e2e-test resourceVersion: "13399" uid: e6cb64b5-1bab-48c6-997f-4dddb3884c8d kind: Event lastTimestamp: "2026-04-20T20:13:47Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.408s (2.409s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-04-20T20:13:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:13:47Z" name: success-200-isvc-127f1-predictor-68b975dccb-dcphk.18a829d043a85356 namespace: kserve-ci-e2e-test resourceVersion: "13690" uid: 22eb05c0-91d6-4e39-8b1a-6b9ed80ec7d2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:13:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-127f1-predictor-68b975dccb-dcphk namespace: kserve-ci-e2e-test resourceVersion: "13399" uid: e6cb64b5-1bab-48c6-997f-4dddb3884c8d kind: Event lastTimestamp: "2026-04-20T20:13:47Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-20T20:13:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:13:47Z" name: success-200-isvc-127f1-predictor-68b975dccb-dcphk.18a829d04671102b namespace: kserve-ci-e2e-test resourceVersion: "13691" uid: ca3dc675-4cb6-428e-bf9a-4f88dfa1945c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:13:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-127f1-predictor-68b975dccb-dcphk namespace: kserve-ci-e2e-test resourceVersion: "13399" uid: e6cb64b5-1bab-48c6-997f-4dddb3884c8d kind: Event lastTimestamp: "2026-04-20T20:13:47Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:13:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:13:47Z" name: success-200-isvc-127f1-predictor-68b975dccb-dcphk.18a829d046e0732c namespace: kserve-ci-e2e-test resourceVersion: "13692" uid: aac19e29-2547-408b-8d00-cb46843f088c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-20T20:13:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-127f1-predictor-68b975dccb-dcphk namespace: kserve-ci-e2e-test resourceVersion: "13399" uid: e6cb64b5-1bab-48c6-997f-4dddb3884c8d kind: Event lastTimestamp: "2026-04-20T20:14:24Z" message: 'Readiness probe failed: dial tcp 10.132.0.18:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-20T20:13:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:14:24Z" name: success-200-isvc-127f1-predictor-68b975dccb-dcphk.18a829d07640bf8d namespace: kserve-ci-e2e-test resourceVersion: "14040" uid: 215433f6-3c3e-4501-9037-29640ef9feb7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-127f1-predictor-68b975dccb-dcphk namespace: kserve-ci-e2e-test resourceVersion: "13399" uid: e6cb64b5-1bab-48c6-997f-4dddb3884c8d kind: Event lastTimestamp: "2026-04-20T20:15:00Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-20T20:15:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:15:00Z" name: success-200-isvc-127f1-predictor-68b975dccb-dcphk.18a829e139195654 namespace: kserve-ci-e2e-test resourceVersion: "14448" uid: 5ef158aa-81b0-481a-bcba-d218c7ea29f8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-127f1-predictor-68b975dccb-dcphk namespace: kserve-ci-e2e-test resourceVersion: "13399" uid: e6cb64b5-1bab-48c6-997f-4dddb3884c8d kind: Event lastTimestamp: "2026-04-20T20:15:00Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:15:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:15:00Z" name: success-200-isvc-127f1-predictor-68b975dccb-dcphk.18a829e13919f31f namespace: kserve-ci-e2e-test resourceVersion: "14452" uid: a2f7d9ab-323d-4250-8a59-e36b0e9b1c2a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:13:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-127f1-predictor-68b975dccb namespace: kserve-ci-e2e-test resourceVersion: "13390" uid: 954eb846-c937-4566-a2d4-1ad1c71434e5 kind: Event lastTimestamp: "2026-04-20T20:13:30Z" message: 'Created pod: success-200-isvc-127f1-predictor-68b975dccb-dcphk' metadata: creationTimestamp: "2026-04-20T20:13:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:13:30Z" name: success-200-isvc-127f1-predictor-68b975dccb.18a829cc3b6d584a namespace: kserve-ci-e2e-test resourceVersion: "13398" uid: 64de9c74-7263-44a0-b831-00d85569ae3b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:13:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-127f1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13387" uid: ffd191b3-92a2-4bbc-aa20-df9c067a7b28 kind: Event lastTimestamp: "2026-04-20T20:13:30Z" message: Scaled up replica set success-200-isvc-127f1-predictor-68b975dccb from 0 to 1 metadata: creationTimestamp: "2026-04-20T20:13:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:13:30Z" name: success-200-isvc-127f1-predictor.18a829cc39f865ab namespace: kserve-ci-e2e-test resourceVersion: "13391" uid: 1350368e-e681-4775-86fc-388e0c0fe542 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:13:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-127f1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13389" uid: e303c0e8-1c74-4497-8f4d-372b0002a5eb kind: Event lastTimestamp: "2026-04-20T20:14:00Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:13:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:14:00Z" name: success-200-isvc-127f1-predictor.18a829cfb8a495f2 namespace: kserve-ci-e2e-test resourceVersion: "13837" uid: e088cfdc-8070-43a9-a93a-a0ea2707f9b9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:13:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-127f1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13389" uid: e303c0e8-1c74-4497-8f4d-372b0002a5eb kind: Event lastTimestamp: "2026-04-20T20:14:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:13:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:14:00Z" name: success-200-isvc-127f1-predictor.18a829cfb8f7f9c4 namespace: kserve-ci-e2e-test resourceVersion: "13838" uid: 1e72605f-e147-4269-8b44-3b13af694202 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-20T20:14:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-127f1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13651" uid: e303c0e8-1c74-4497-8f4d-372b0002a5eb kind: Event lastTimestamp: "2026-04-20T20:15:00Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:14:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:15:00Z" name: success-200-isvc-127f1-predictor.18a829d6b681dfc4 namespace: kserve-ci-e2e-test resourceVersion: "14402" uid: dff8ba4c-cfba-4887-90a8-82fa1f4d9a23 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-20T20:14:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-127f1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13651" uid: e303c0e8-1c74-4497-8f4d-372b0002a5eb kind: Event lastTimestamp: "2026-04-20T20:15:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:14:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:15:00Z" name: success-200-isvc-127f1-predictor.18a829d6b6f3d5d3 namespace: kserve-ci-e2e-test resourceVersion: "14404" uid: c0a8183c-8567-439c-8c66-b3b0fcdb3f81 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-20T20:13:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-127f1 namespace: kserve-ci-e2e-test resourceVersion: "13370" uid: 0b582940-0db7-4735-9e1c-132919167de9 kind: Event lastTimestamp: "2026-04-20T20:14:34Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-20T20:13:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:14:34Z" name: success-200-isvc-127f1.18a829cc3a02cace namespace: kserve-ci-e2e-test resourceVersion: "14134" uid: 541f1a03-cc75-43a9-9d81-a3359ce1bf15 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:14:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-127f1 namespace: kserve-ci-e2e-test resourceVersion: "14132" uid: 0b582940-0db7-4735-9e1c-132919167de9 kind: Event lastTimestamp: "2026-04-20T20:14:34Z" message: InferenceService [success-200-isvc-127f1] is Ready metadata: creationTimestamp: "2026-04-20T20:14:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:14:34Z" name: success-200-isvc-127f1.18a829db311ede2b namespace: kserve-ci-e2e-test resourceVersion: "14133" uid: 8ddb9c77-ed33-49d2-b24d-668cbe4f9ed2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-20T20:25:25.670845Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-38d71-predictor-c6f86d4b4-w2stk namespace: kserve-ci-e2e-test resourceVersion: "19746" uid: e37a9c1b-d07a-4b7c-8cf3-d03f4a9040a8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-38d71-predictor-c6f86d4b4-w2stk to ip-10-0-140-19.ec2.internal metadata: creationTimestamp: "2026-04-20T20:25:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-20T20:25:25Z" name: success-200-isvc-38d71-predictor-c6f86d4b4-w2stk.18a82a72d41b3894 namespace: kserve-ci-e2e-test resourceVersion: "19753" uid: fae63f63-35f7-4b91-8a0b-ec54b50385db reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58dbb994fc-hbcnb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:25Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-38d71-predictor-c6f86d4b4-w2stk namespace: kserve-ci-e2e-test resourceVersion: "19749" uid: e37a9c1b-d07a-4b7c-8cf3-d03f4a9040a8 kind: Event lastTimestamp: "2026-04-20T20:25:25Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-38d71-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-20T20:25:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:25:25Z" name: success-200-isvc-38d71-predictor-c6f86d4b4-w2stk.18a82a72dff029c2 namespace: kserve-ci-e2e-test resourceVersion: "19800" uid: 33e9e0be-92d0-416d-8c79-516693d8ab40 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:26Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-38d71-predictor-c6f86d4b4-w2stk namespace: kserve-ci-e2e-test resourceVersion: "19768" uid: e37a9c1b-d07a-4b7c-8cf3-d03f4a9040a8 kind: Event lastTimestamp: "2026-04-20T20:25:26Z" message: Add eth0 [10.132.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-20T20:25:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-20T20:25:26Z" name: success-200-isvc-38d71-predictor-c6f86d4b4-w2stk.18a82a7310e1ea42 namespace: kserve-ci-e2e-test resourceVersion: "19819" uid: 5fd54252-1566-4663-bcc3-b6d6619ffcde reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-38d71-predictor-c6f86d4b4-w2stk namespace: kserve-ci-e2e-test resourceVersion: "19749" uid: e37a9c1b-d07a-4b7c-8cf3-d03f4a9040a8 kind: Event lastTimestamp: "2026-04-20T20:25:26Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-20T20:25:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:25:26Z" name: success-200-isvc-38d71-predictor-c6f86d4b4-w2stk.18a82a7312879880 namespace: kserve-ci-e2e-test resourceVersion: "19821" uid: 5737598d-64ec-46d6-becb-4acb1147242b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-38d71-predictor-c6f86d4b4-w2stk namespace: kserve-ci-e2e-test resourceVersion: "19749" uid: e37a9c1b-d07a-4b7c-8cf3-d03f4a9040a8 kind: Event lastTimestamp: "2026-04-20T20:25:26Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-20T20:25:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:25:26Z" name: success-200-isvc-38d71-predictor-c6f86d4b4-w2stk.18a82a73159f2ccc namespace: kserve-ci-e2e-test resourceVersion: "19822" uid: 6ba0a129-07bf-4df2-9a86-0f06bfa62e24 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-38d71-predictor-c6f86d4b4-w2stk namespace: kserve-ci-e2e-test resourceVersion: "19749" uid: e37a9c1b-d07a-4b7c-8cf3-d03f4a9040a8 kind: Event lastTimestamp: "2026-04-20T20:25:26Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-20T20:25:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:25:26Z" name: success-200-isvc-38d71-predictor-c6f86d4b4-w2stk.18a82a7316225290 namespace: kserve-ci-e2e-test resourceVersion: "19823" uid: b8f755d8-e6ec-4046-bf86-6512e135f94f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-38d71-predictor-c6f86d4b4-w2stk namespace: kserve-ci-e2e-test resourceVersion: "19749" uid: e37a9c1b-d07a-4b7c-8cf3-d03f4a9040a8 kind: Event lastTimestamp: "2026-04-20T20:25:26Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-20T20:25:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:25:26Z" name: success-200-isvc-38d71-predictor-c6f86d4b4-w2stk.18a82a73162f04c6 namespace: kserve-ci-e2e-test resourceVersion: "19824" uid: 6b5815a7-4c15-42b9-9164-7925eb90133e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-38d71-predictor-c6f86d4b4-w2stk namespace: kserve-ci-e2e-test resourceVersion: "19749" uid: e37a9c1b-d07a-4b7c-8cf3-d03f4a9040a8 kind: Event lastTimestamp: "2026-04-20T20:25:26Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-20T20:25:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:25:26Z" name: success-200-isvc-38d71-predictor-c6f86d4b4-w2stk.18a82a7318b8133a namespace: kserve-ci-e2e-test resourceVersion: "19825" uid: 6fd86537-83c7-4755-93bb-64db8fdb9b15 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-38d71-predictor-c6f86d4b4-w2stk namespace: kserve-ci-e2e-test resourceVersion: "19749" uid: e37a9c1b-d07a-4b7c-8cf3-d03f4a9040a8 kind: Event lastTimestamp: "2026-04-20T20:25:26Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:25:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:25:26Z" name: success-200-isvc-38d71-predictor-c6f86d4b4-w2stk.18a82a73196c8805 namespace: kserve-ci-e2e-test resourceVersion: "19826" uid: 10187891-fbe7-4296-a086-98010e92ea05 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-20T20:25:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-38d71-predictor-c6f86d4b4-w2stk namespace: kserve-ci-e2e-test resourceVersion: "19749" uid: e37a9c1b-d07a-4b7c-8cf3-d03f4a9040a8 kind: Event lastTimestamp: "2026-04-20T20:26:04Z" message: 'Readiness probe failed: dial tcp 10.132.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-20T20:25:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:26:04Z" name: success-200-isvc-38d71-predictor-c6f86d4b4-w2stk.18a82a7372f0aaa6 namespace: kserve-ci-e2e-test resourceVersion: "20205" uid: 42fce68f-3afc-4237-9474-c1e6ec3138e4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:34:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-38d71-predictor-c6f86d4b4-w2stk namespace: kserve-ci-e2e-test resourceVersion: "19749" uid: e37a9c1b-d07a-4b7c-8cf3-d03f4a9040a8 kind: Event lastTimestamp: "2026-04-20T20:34:40Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-20T20:34:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:34:40Z" name: success-200-isvc-38d71-predictor-c6f86d4b4-w2stk.18a82af4003fa7ee namespace: kserve-ci-e2e-test resourceVersion: "24087" uid: 9b49042c-5949-48b6-ae02-d7c3501e4e4e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:34:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-38d71-predictor-c6f86d4b4-w2stk namespace: kserve-ci-e2e-test resourceVersion: "19749" uid: e37a9c1b-d07a-4b7c-8cf3-d03f4a9040a8 kind: Event lastTimestamp: "2026-04-20T20:34:40Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:34:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:34:40Z" name: success-200-isvc-38d71-predictor-c6f86d4b4-w2stk.18a82af4004083a6 namespace: kserve-ci-e2e-test resourceVersion: "24091" uid: 1678a22a-680b-4966-af48-9d208cb8d327 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-38d71-predictor-c6f86d4b4 namespace: kserve-ci-e2e-test resourceVersion: "19734" uid: 11e67990-f163-4729-a180-71ff9024bdd6 kind: Event lastTimestamp: "2026-04-20T20:25:25Z" message: 'Created pod: success-200-isvc-38d71-predictor-c6f86d4b4-w2stk' metadata: creationTimestamp: "2026-04-20T20:25:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:25:25Z" name: success-200-isvc-38d71-predictor-c6f86d4b4.18a82a72d39cb215 namespace: kserve-ci-e2e-test resourceVersion: "19748" uid: 4bf38bef-c0b3-4c65-b487-98ddccf83b89 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-38d71-predictor namespace: kserve-ci-e2e-test resourceVersion: "19727" uid: 514a161b-92fd-4b5e-af04-7c79e414bc3b kind: Event lastTimestamp: "2026-04-20T20:25:25Z" message: Scaled up replica set success-200-isvc-38d71-predictor-c6f86d4b4 from 0 to 1 metadata: creationTimestamp: "2026-04-20T20:25:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:25:25Z" name: success-200-isvc-38d71-predictor.18a82a72d1992a91 namespace: kserve-ci-e2e-test resourceVersion: "19736" uid: c4bd6574-2645-4470-8b32-53e5fdc3cc55 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-38d71-predictor namespace: kserve-ci-e2e-test resourceVersion: "19733" uid: 4658f1aa-1531-4c49-b1f9-4287b7ded108 kind: Event lastTimestamp: "2026-04-20T20:25:40Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:25:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:25:40Z" name: success-200-isvc-38d71-predictor.18a82a7650470d0c namespace: kserve-ci-e2e-test resourceVersion: "19953" uid: 325c8955-45c2-4c05-9f0e-bc01aaf7a2f6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-38d71-predictor namespace: kserve-ci-e2e-test resourceVersion: "19733" uid: 4658f1aa-1531-4c49-b1f9-4287b7ded108 kind: Event lastTimestamp: "2026-04-20T20:25:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:25:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:25:40Z" name: success-200-isvc-38d71-predictor.18a82a7650a9a706 namespace: kserve-ci-e2e-test resourceVersion: "19955" uid: 5cae78a9-7a3c-42dc-9e76-158fec8b27c9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-20T20:25:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-38d71-predictor namespace: kserve-ci-e2e-test resourceVersion: "19954" uid: 4658f1aa-1531-4c49-b1f9-4287b7ded108 kind: Event lastTimestamp: "2026-04-20T20:26:40Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:25:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:26:40Z" name: success-200-isvc-38d71-predictor.18a82a79cf68baa7 namespace: kserve-ci-e2e-test resourceVersion: "20694" uid: d4ab0744-c981-47c9-80ae-69095d9a8c4a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-20T20:25:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-38d71-predictor namespace: kserve-ci-e2e-test resourceVersion: "19954" uid: 4658f1aa-1531-4c49-b1f9-4287b7ded108 kind: Event lastTimestamp: "2026-04-20T20:26:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:25:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:26:40Z" name: success-200-isvc-38d71-predictor.18a82a79cfcaa5c6 namespace: kserve-ci-e2e-test resourceVersion: "20696" uid: 5399d25a-9fc6-4c87-b5aa-913a651abb68 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-20T20:25:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-38d71 namespace: kserve-ci-e2e-test resourceVersion: "19714" uid: f38422af-d572-4c12-b6ea-9561f13a1e41 kind: Event lastTimestamp: "2026-04-20T20:26:14Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-20T20:25:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:26:14Z" name: success-200-isvc-38d71.18a82a72d1af8331 namespace: kserve-ci-e2e-test resourceVersion: "20446" uid: c87bc6bc-32b0-4c09-a5c3-48ca3ef56dcf reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-38d71 namespace: kserve-ci-e2e-test resourceVersion: "19714" uid: f38422af-d572-4c12-b6ea-9561f13a1e41 kind: Event lastTimestamp: "2026-04-20T20:25:25Z" message: 'Failed to update status for InferenceService "success-200-isvc-38d71": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-38d71": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-20T20:25:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:25:25Z" name: success-200-isvc-38d71.18a82a72d21aadf9 namespace: kserve-ci-e2e-test resourceVersion: "19741" uid: ce89a8bf-f328-4443-93ac-2a0868c40bdb reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-38d71 namespace: kserve-ci-e2e-test resourceVersion: "19714" uid: f38422af-d572-4c12-b6ea-9561f13a1e41 kind: Event lastTimestamp: "2026-04-20T20:25:25Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-38d71": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-20T20:25:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:25:25Z" name: success-200-isvc-38d71.18a82a72d21aff2e namespace: kserve-ci-e2e-test resourceVersion: "19742" uid: 8673ea89-c607-49ca-9d82-b7e6597343e5 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:26:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-38d71 namespace: kserve-ci-e2e-test resourceVersion: "20444" uid: f38422af-d572-4c12-b6ea-9561f13a1e41 kind: Event lastTimestamp: "2026-04-20T20:26:14Z" message: InferenceService [success-200-isvc-38d71] is Ready metadata: creationTimestamp: "2026-04-20T20:26:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:26:14Z" name: success-200-isvc-38d71.18a82a7e2dd1b95b namespace: kserve-ci-e2e-test resourceVersion: "20445" uid: 2c14e0a3-51c7-463a-8395-8426a766cfbd reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-20T20:35:50.822833Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-57557-predictor-7cdcf47b7b-75nzd namespace: kserve-ci-e2e-test resourceVersion: "25005" uid: bf768051-677e-4649-bf94-78323dfd123e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-57557-predictor-7cdcf47b7b-75nzd to ip-10-0-140-19.ec2.internal metadata: creationTimestamp: "2026-04-20T20:35:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-20T20:35:50Z" name: success-200-isvc-57557-predictor-7cdcf47b7b-75nzd.18a82b0462118cf7 namespace: kserve-ci-e2e-test resourceVersion: "25010" uid: 927c0640-c080-4542-a696-677e401da934 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58dbb994fc-hbcnb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:51Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-57557-predictor-7cdcf47b7b-75nzd namespace: kserve-ci-e2e-test resourceVersion: "25007" uid: bf768051-677e-4649-bf94-78323dfd123e kind: Event lastTimestamp: "2026-04-20T20:35:51Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-57557-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-20T20:35:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:35:51Z" name: success-200-isvc-57557-predictor-7cdcf47b7b-75nzd.18a82b047013f67b namespace: kserve-ci-e2e-test resourceVersion: "25047" uid: 415d33f7-dcd2-44f0-b01e-53c9d9840d2d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:51Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-57557-predictor-7cdcf47b7b-75nzd namespace: kserve-ci-e2e-test resourceVersion: "25015" uid: bf768051-677e-4649-bf94-78323dfd123e kind: Event lastTimestamp: "2026-04-20T20:35:51Z" message: Add eth0 [10.132.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-20T20:35:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-20T20:35:51Z" name: success-200-isvc-57557-predictor-7cdcf47b7b-75nzd.18a82b049e708f2e namespace: kserve-ci-e2e-test resourceVersion: "25066" uid: f47d6f3f-1e21-4ca0-9f67-a8f295ab994f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-57557-predictor-7cdcf47b7b-75nzd namespace: kserve-ci-e2e-test resourceVersion: "25007" uid: bf768051-677e-4649-bf94-78323dfd123e kind: Event lastTimestamp: "2026-04-20T20:35:51Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-20T20:35:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:35:51Z" name: success-200-isvc-57557-predictor-7cdcf47b7b-75nzd.18a82b049fae3c86 namespace: kserve-ci-e2e-test resourceVersion: "25068" uid: 8c2c0db7-fa5e-4ff5-9416-30e1c80d34f1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-57557-predictor-7cdcf47b7b-75nzd namespace: kserve-ci-e2e-test resourceVersion: "25007" uid: bf768051-677e-4649-bf94-78323dfd123e kind: Event lastTimestamp: "2026-04-20T20:35:51Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-20T20:35:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:35:51Z" name: success-200-isvc-57557-predictor-7cdcf47b7b-75nzd.18a82b04a28bc435 namespace: kserve-ci-e2e-test resourceVersion: "25069" uid: 6da1c53d-6665-4b98-82f0-2ed620b8b9cf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-57557-predictor-7cdcf47b7b-75nzd namespace: kserve-ci-e2e-test resourceVersion: "25007" uid: bf768051-677e-4649-bf94-78323dfd123e kind: Event lastTimestamp: "2026-04-20T20:35:51Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-20T20:35:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:35:51Z" name: success-200-isvc-57557-predictor-7cdcf47b7b-75nzd.18a82b04a2f98ccf namespace: kserve-ci-e2e-test resourceVersion: "25070" uid: 6fbcd0d0-4066-430a-8570-1a25e33a4ca9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-57557-predictor-7cdcf47b7b-75nzd namespace: kserve-ci-e2e-test resourceVersion: "25007" uid: bf768051-677e-4649-bf94-78323dfd123e kind: Event lastTimestamp: "2026-04-20T20:35:51Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-20T20:35:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:35:51Z" name: success-200-isvc-57557-predictor-7cdcf47b7b-75nzd.18a82b04a3047be5 namespace: kserve-ci-e2e-test resourceVersion: "25071" uid: be9c9044-97a7-4cc8-8f05-c3b4e1729eb0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-57557-predictor-7cdcf47b7b-75nzd namespace: kserve-ci-e2e-test resourceVersion: "25007" uid: bf768051-677e-4649-bf94-78323dfd123e kind: Event lastTimestamp: "2026-04-20T20:35:51Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-20T20:35:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:35:51Z" name: success-200-isvc-57557-predictor-7cdcf47b7b-75nzd.18a82b04a5d03e72 namespace: kserve-ci-e2e-test resourceVersion: "25073" uid: 16bd9b61-b04b-4268-b090-4820ac547d31 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-57557-predictor-7cdcf47b7b-75nzd namespace: kserve-ci-e2e-test resourceVersion: "25007" uid: bf768051-677e-4649-bf94-78323dfd123e kind: Event lastTimestamp: "2026-04-20T20:35:51Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:35:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:35:51Z" name: success-200-isvc-57557-predictor-7cdcf47b7b-75nzd.18a82b04a649d278 namespace: kserve-ci-e2e-test resourceVersion: "25074" uid: 41d4299a-897a-4a26-b72b-e753a27e8149 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-20T20:35:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-57557-predictor-7cdcf47b7b-75nzd namespace: kserve-ci-e2e-test resourceVersion: "25007" uid: bf768051-677e-4649-bf94-78323dfd123e kind: Event lastTimestamp: "2026-04-20T20:36:28Z" message: 'Readiness probe failed: dial tcp 10.132.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-20T20:35:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:36:28Z" name: success-200-isvc-57557-predictor-7cdcf47b7b-75nzd.18a82b04af8c0dad namespace: kserve-ci-e2e-test resourceVersion: "25444" uid: fd9a8a81-f475-4911-8498-9f80be99b115 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:45:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-57557-predictor-7cdcf47b7b-75nzd namespace: kserve-ci-e2e-test resourceVersion: "25007" uid: bf768051-677e-4649-bf94-78323dfd123e kind: Event lastTimestamp: "2026-04-20T20:45:05Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-20T20:45:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:45:05Z" name: success-200-isvc-57557-predictor-7cdcf47b7b-75nzd.18a82b858b52c079 namespace: kserve-ci-e2e-test resourceVersion: "29428" uid: 022809e0-1305-47a6-a89f-275a688f271b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:45:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-57557-predictor-7cdcf47b7b-75nzd namespace: kserve-ci-e2e-test resourceVersion: "25007" uid: bf768051-677e-4649-bf94-78323dfd123e kind: Event lastTimestamp: "2026-04-20T20:45:05Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:45:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:45:05Z" name: success-200-isvc-57557-predictor-7cdcf47b7b-75nzd.18a82b858b535502 namespace: kserve-ci-e2e-test resourceVersion: "29429" uid: 4e8a09da-f328-4a7a-a562-5b5d955fefc2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:45:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-57557-predictor-7cdcf47b7b-75nzd namespace: kserve-ci-e2e-test resourceVersion: "25007" uid: bf768051-677e-4649-bf94-78323dfd123e kind: Event lastTimestamp: "2026-04-20T20:45:08Z" message: 'Readiness probe failed: Get "https://10.132.0.28:8643/healthz": dial tcp 10.132.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-20T20:45:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:45:08Z" name: success-200-isvc-57557-predictor-7cdcf47b7b-75nzd.18a82b8623eb0754 namespace: kserve-ci-e2e-test resourceVersion: "29468" uid: e1c4c9ae-489f-4edb-8d11-49046e0c45e8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:45:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-57557-predictor-7cdcf47b7b-75nzd namespace: kserve-ci-e2e-test resourceVersion: "25007" uid: bf768051-677e-4649-bf94-78323dfd123e kind: Event lastTimestamp: "2026-04-20T20:45:09Z" message: 'Readiness probe failed: dial tcp 10.132.0.28:8080: i/o timeout' metadata: creationTimestamp: "2026-04-20T20:45:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:45:09Z" name: success-200-isvc-57557-predictor-7cdcf47b7b-75nzd.18a82b865fd2f588 namespace: kserve-ci-e2e-test resourceVersion: "29480" uid: 89748284-b5f1-4b48-a4e1-4dcf3e42f5d3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-57557-predictor-7cdcf47b7b namespace: kserve-ci-e2e-test resourceVersion: "24991" uid: e6540362-f4c0-4179-8058-ce7828cdd2fb kind: Event lastTimestamp: "2026-04-20T20:35:50Z" message: 'Created pod: success-200-isvc-57557-predictor-7cdcf47b7b-75nzd' metadata: creationTimestamp: "2026-04-20T20:35:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:35:50Z" name: success-200-isvc-57557-predictor-7cdcf47b7b.18a82b0461ac57b1 namespace: kserve-ci-e2e-test resourceVersion: "25006" uid: bf674276-7422-4569-ad07-2d34e47aacc1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-57557-predictor namespace: kserve-ci-e2e-test resourceVersion: "24986" uid: 58473a71-20c1-4715-8c82-24170bb4029a kind: Event lastTimestamp: "2026-04-20T20:35:50Z" message: Scaled up replica set success-200-isvc-57557-predictor-7cdcf47b7b from 0 to 1 metadata: creationTimestamp: "2026-04-20T20:35:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:35:50Z" name: success-200-isvc-57557-predictor.18a82b045f557b7d namespace: kserve-ci-e2e-test resourceVersion: "24995" uid: de45baab-7343-43d0-81d7-c1b2b81968e9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-20T20:36:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-57557-predictor namespace: kserve-ci-e2e-test resourceVersion: "24990" uid: 1b680a50-c104-43c0-b69c-149f36dee676 kind: Event lastTimestamp: "2026-04-20T20:36:50Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:36:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:36:50Z" name: success-200-isvc-57557-predictor.18a82b07de005069 namespace: kserve-ci-e2e-test resourceVersion: "25630" uid: 815c1fd0-a930-47bb-88b1-5e610252bd21 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-20T20:36:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-57557-predictor namespace: kserve-ci-e2e-test resourceVersion: "24990" uid: 1b680a50-c104-43c0-b69c-149f36dee676 kind: Event lastTimestamp: "2026-04-20T20:36:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:36:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:36:50Z" name: success-200-isvc-57557-predictor.18a82b07de66a660 namespace: kserve-ci-e2e-test resourceVersion: "25631" uid: bc8b6508-9e82-4c13-83f9-2804bfcc58b3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:36:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-57557-predictor namespace: kserve-ci-e2e-test resourceVersion: "25209" uid: 1b680a50-c104-43c0-b69c-149f36dee676 kind: Event lastTimestamp: "2026-04-20T20:36:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:36:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:36:20Z" name: success-200-isvc-57557-predictor.18a82b0b5d31ce49 namespace: kserve-ci-e2e-test resourceVersion: "25370" uid: c7116257-7f1b-4b5a-8b88-334cf8d87df0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:36:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-57557-predictor namespace: kserve-ci-e2e-test resourceVersion: "25209" uid: 1b680a50-c104-43c0-b69c-149f36dee676 kind: Event lastTimestamp: "2026-04-20T20:36:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:36:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:36:20Z" name: success-200-isvc-57557-predictor.18a82b0b5d93b51c namespace: kserve-ci-e2e-test resourceVersion: "25372" uid: f06942ba-8014-4ae8-bdfa-771801dd9e85 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-20T20:35:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-57557 namespace: kserve-ci-e2e-test resourceVersion: "24963" uid: 1c4fafeb-d8c3-4e28-acd2-faef162bbbee kind: Event lastTimestamp: "2026-04-20T20:36:38Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-20T20:35:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:36:38Z" name: success-200-isvc-57557.18a82b045f5ebe1f namespace: kserve-ci-e2e-test resourceVersion: "25534" uid: 954ff5c2-3dc8-44c5-8e3e-734364ab27ca reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-57557 namespace: kserve-ci-e2e-test resourceVersion: "24963" uid: 1c4fafeb-d8c3-4e28-acd2-faef162bbbee kind: Event lastTimestamp: "2026-04-20T20:35:50Z" message: 'Failed to update status for InferenceService "success-200-isvc-57557": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-57557": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-20T20:35:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:35:50Z" name: success-200-isvc-57557.18a82b04602fdaa6 namespace: kserve-ci-e2e-test resourceVersion: "24999" uid: 1a9a0a7a-72b1-4e85-995d-16c0a3c660c8 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-57557 namespace: kserve-ci-e2e-test resourceVersion: "24963" uid: 1c4fafeb-d8c3-4e28-acd2-faef162bbbee kind: Event lastTimestamp: "2026-04-20T20:35:50Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-57557": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-20T20:35:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:35:50Z" name: success-200-isvc-57557.18a82b0460303903 namespace: kserve-ci-e2e-test resourceVersion: "25001" uid: 4c1c88a8-6eec-4744-b85c-fc944d6f1bbf reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:36:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-57557 namespace: kserve-ci-e2e-test resourceVersion: "25532" uid: 1c4fafeb-d8c3-4e28-acd2-faef162bbbee kind: Event lastTimestamp: "2026-04-20T20:36:38Z" message: InferenceService [success-200-isvc-57557] is Ready metadata: creationTimestamp: "2026-04-20T20:36:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:36:38Z" name: success-200-isvc-57557.18a82b0f69e0783b namespace: kserve-ci-e2e-test resourceVersion: "25533" uid: 9a72fae3-ceda-4bcb-90ce-09195fff9e96 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-20T20:35:20.335320Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-60a5c-predictor-5fbfb7dd4-hv76t namespace: kserve-ci-e2e-test resourceVersion: "24546" uid: 103ac5eb-fe4a-4dd4-9fda-e59c1d9263d6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-60a5c-predictor-5fbfb7dd4-hv76t to ip-10-0-140-19.ec2.internal metadata: creationTimestamp: "2026-04-20T20:35:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-20T20:35:20Z" name: success-200-isvc-60a5c-predictor-5fbfb7dd4-hv76t.18a82afd48df0b04 namespace: kserve-ci-e2e-test resourceVersion: "24554" uid: 5415b70c-85e0-44c4-a452-34e95c229e65 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58dbb994fc-hbcnb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:20Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-60a5c-predictor-5fbfb7dd4-hv76t namespace: kserve-ci-e2e-test resourceVersion: "24549" uid: 103ac5eb-fe4a-4dd4-9fda-e59c1d9263d6 kind: Event lastTimestamp: "2026-04-20T20:35:20Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-60a5c-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-20T20:35:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:35:20Z" name: success-200-isvc-60a5c-predictor-5fbfb7dd4-hv76t.18a82afd5599b49e namespace: kserve-ci-e2e-test resourceVersion: "24598" uid: da89faea-7e02-49d9-8676-a1f3c6616195 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:21Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-60a5c-predictor-5fbfb7dd4-hv76t namespace: kserve-ci-e2e-test resourceVersion: "24568" uid: 103ac5eb-fe4a-4dd4-9fda-e59c1d9263d6 kind: Event lastTimestamp: "2026-04-20T20:35:21Z" message: Add eth0 [10.132.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-20T20:35:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-20T20:35:21Z" name: success-200-isvc-60a5c-predictor-5fbfb7dd4-hv76t.18a82afd859202e5 namespace: kserve-ci-e2e-test resourceVersion: "24611" uid: a3c4b3fb-13c8-488a-adbc-298e31e34cc7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-60a5c-predictor-5fbfb7dd4-hv76t namespace: kserve-ci-e2e-test resourceVersion: "24549" uid: 103ac5eb-fe4a-4dd4-9fda-e59c1d9263d6 kind: Event lastTimestamp: "2026-04-20T20:35:21Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-20T20:35:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:35:21Z" name: success-200-isvc-60a5c-predictor-5fbfb7dd4-hv76t.18a82afd86d731a1 namespace: kserve-ci-e2e-test resourceVersion: "24613" uid: 9314df28-89ea-44ca-b558-0b9c5694b53f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-60a5c-predictor-5fbfb7dd4-hv76t namespace: kserve-ci-e2e-test resourceVersion: "24549" uid: 103ac5eb-fe4a-4dd4-9fda-e59c1d9263d6 kind: Event lastTimestamp: "2026-04-20T20:35:21Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-20T20:35:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:35:21Z" name: success-200-isvc-60a5c-predictor-5fbfb7dd4-hv76t.18a82afd8a13635c namespace: kserve-ci-e2e-test resourceVersion: "24614" uid: 2dfd8144-f33d-4765-9f00-de57c87d725b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-60a5c-predictor-5fbfb7dd4-hv76t namespace: kserve-ci-e2e-test resourceVersion: "24549" uid: 103ac5eb-fe4a-4dd4-9fda-e59c1d9263d6 kind: Event lastTimestamp: "2026-04-20T20:35:21Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-20T20:35:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:35:21Z" name: success-200-isvc-60a5c-predictor-5fbfb7dd4-hv76t.18a82afd8abb6056 namespace: kserve-ci-e2e-test resourceVersion: "24615" uid: 12ab9028-5bc9-4b06-b137-f9d5d9a9204b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-60a5c-predictor-5fbfb7dd4-hv76t namespace: kserve-ci-e2e-test resourceVersion: "24549" uid: 103ac5eb-fe4a-4dd4-9fda-e59c1d9263d6 kind: Event lastTimestamp: "2026-04-20T20:35:21Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-20T20:35:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:35:21Z" name: success-200-isvc-60a5c-predictor-5fbfb7dd4-hv76t.18a82afd8ac7d096 namespace: kserve-ci-e2e-test resourceVersion: "24616" uid: 6a4d6494-b65a-4d9f-93bb-192c1bb95985 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-60a5c-predictor-5fbfb7dd4-hv76t namespace: kserve-ci-e2e-test resourceVersion: "24549" uid: 103ac5eb-fe4a-4dd4-9fda-e59c1d9263d6 kind: Event lastTimestamp: "2026-04-20T20:35:21Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-20T20:35:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:35:21Z" name: success-200-isvc-60a5c-predictor-5fbfb7dd4-hv76t.18a82afd8e338858 namespace: kserve-ci-e2e-test resourceVersion: "24621" uid: 4018f686-5a9c-423a-b572-7cdde413f5d1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-60a5c-predictor-5fbfb7dd4-hv76t namespace: kserve-ci-e2e-test resourceVersion: "24549" uid: 103ac5eb-fe4a-4dd4-9fda-e59c1d9263d6 kind: Event lastTimestamp: "2026-04-20T20:35:21Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:35:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:35:21Z" name: success-200-isvc-60a5c-predictor-5fbfb7dd4-hv76t.18a82afd8ea1afeb namespace: kserve-ci-e2e-test resourceVersion: "24622" uid: 3e369df7-6252-4997-90d6-7ac3cafc9142 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-20T20:35:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-60a5c-predictor-5fbfb7dd4-hv76t namespace: kserve-ci-e2e-test resourceVersion: "24549" uid: 103ac5eb-fe4a-4dd4-9fda-e59c1d9263d6 kind: Event lastTimestamp: "2026-04-20T20:35:58Z" message: 'Readiness probe failed: dial tcp 10.132.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-20T20:35:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:35:58Z" name: success-200-isvc-60a5c-predictor-5fbfb7dd4-hv76t.18a82afdadf8ff54 namespace: kserve-ci-e2e-test resourceVersion: "25138" uid: 0cd7eb64-8aa9-4f66-934e-a35aa5686641 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:52:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-60a5c-predictor-5fbfb7dd4-hv76t namespace: kserve-ci-e2e-test resourceVersion: "24549" uid: 103ac5eb-fe4a-4dd4-9fda-e59c1d9263d6 kind: Event lastTimestamp: "2026-04-20T20:52:40Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-20T20:52:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:52:40Z" name: success-200-isvc-60a5c-predictor-5fbfb7dd4-hv76t.18a82bef5b1c4692 namespace: kserve-ci-e2e-test resourceVersion: "32584" uid: b04b1676-db3b-4e7d-ae0f-6880c3ea7735 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:52:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-60a5c-predictor-5fbfb7dd4-hv76t namespace: kserve-ci-e2e-test resourceVersion: "24549" uid: 103ac5eb-fe4a-4dd4-9fda-e59c1d9263d6 kind: Event lastTimestamp: "2026-04-20T20:52:40Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:52:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:52:40Z" name: success-200-isvc-60a5c-predictor-5fbfb7dd4-hv76t.18a82bef5b1cc710 namespace: kserve-ci-e2e-test resourceVersion: "32585" uid: 47f04d46-e76a-4c5d-b78a-02bec5be5f09 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-60a5c-predictor-5fbfb7dd4 namespace: kserve-ci-e2e-test resourceVersion: "24533" uid: 7c1891d9-ecd1-4284-a55f-352fa1130fb1 kind: Event lastTimestamp: "2026-04-20T20:35:20Z" message: 'Created pod: success-200-isvc-60a5c-predictor-5fbfb7dd4-hv76t' metadata: creationTimestamp: "2026-04-20T20:35:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:35:20Z" name: success-200-isvc-60a5c-predictor-5fbfb7dd4.18a82afd48544878 namespace: kserve-ci-e2e-test resourceVersion: "24548" uid: 0a1e0a60-f889-4047-bdd6-f3be08de5ee2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-60a5c-predictor namespace: kserve-ci-e2e-test resourceVersion: "24526" uid: 55951348-80ea-43c5-8c1b-6421d55d54b7 kind: Event lastTimestamp: "2026-04-20T20:35:20Z" message: Scaled up replica set success-200-isvc-60a5c-predictor-5fbfb7dd4 from 0 to 1 metadata: creationTimestamp: "2026-04-20T20:35:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:35:20Z" name: success-200-isvc-60a5c-predictor.18a82afd463075f3 namespace: kserve-ci-e2e-test resourceVersion: "24535" uid: eb16d0e4-2964-4c03-8dcd-45e220cda5ef reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-20T20:35:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-60a5c-predictor namespace: kserve-ci-e2e-test resourceVersion: "24532" uid: 140b57f2-fc54-4e1d-ac54-e49f551fe831 kind: Event lastTimestamp: "2026-04-20T20:36:35Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:35:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:36:35Z" name: success-200-isvc-60a5c-predictor.18a82b00c4db72a8 namespace: kserve-ci-e2e-test resourceVersion: "25495" uid: 665bd8fb-4c54-4363-8b71-ca0d701505c0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-20T20:35:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-60a5c-predictor namespace: kserve-ci-e2e-test resourceVersion: "24532" uid: 140b57f2-fc54-4e1d-ac54-e49f551fe831 kind: Event lastTimestamp: "2026-04-20T20:36:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:35:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:36:35Z" name: success-200-isvc-60a5c-predictor.18a82b00c54134d7 namespace: kserve-ci-e2e-test resourceVersion: "25496" uid: cbfb23ad-19c5-4148-8006-153c59066efd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-60a5c-predictor namespace: kserve-ci-e2e-test resourceVersion: "24769" uid: 140b57f2-fc54-4e1d-ac54-e49f551fe831 kind: Event lastTimestamp: "2026-04-20T20:35:50Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:35:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:35:50Z" name: success-200-isvc-60a5c-predictor.18a82b04443900aa namespace: kserve-ci-e2e-test resourceVersion: "24930" uid: 93e464b3-ee28-412d-be78-1a6d946d4d40 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-60a5c-predictor namespace: kserve-ci-e2e-test resourceVersion: "24769" uid: 140b57f2-fc54-4e1d-ac54-e49f551fe831 kind: Event lastTimestamp: "2026-04-20T20:35:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:35:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:35:50Z" name: success-200-isvc-60a5c-predictor.18a82b04449a3335 namespace: kserve-ci-e2e-test resourceVersion: "24932" uid: 78e9a414-97b6-42f1-875b-a0066506c3a2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-20T20:35:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-60a5c namespace: kserve-ci-e2e-test resourceVersion: "24512" uid: 7f7aa29e-9984-46e1-8cae-fe52c987f531 kind: Event lastTimestamp: "2026-04-20T20:36:08Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-20T20:35:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:36:08Z" name: success-200-isvc-60a5c.18a82afd4635e63e namespace: kserve-ci-e2e-test resourceVersion: "25240" uid: 43f79926-14fc-41ed-b656-40b378ac0e54 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-60a5c namespace: kserve-ci-e2e-test resourceVersion: "24512" uid: 7f7aa29e-9984-46e1-8cae-fe52c987f531 kind: Event lastTimestamp: "2026-04-20T20:35:20Z" message: 'Failed to update status for InferenceService "success-200-isvc-60a5c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-60a5c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-20T20:35:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:35:20Z" name: success-200-isvc-60a5c.18a82afd46c4085a namespace: kserve-ci-e2e-test resourceVersion: "24539" uid: 7cf9516d-5256-42cd-a0e4-25a74a3e9c27 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-60a5c namespace: kserve-ci-e2e-test resourceVersion: "24512" uid: 7f7aa29e-9984-46e1-8cae-fe52c987f531 kind: Event lastTimestamp: "2026-04-20T20:35:20Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-60a5c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-20T20:35:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:35:20Z" name: success-200-isvc-60a5c.18a82afd46c45e2d namespace: kserve-ci-e2e-test resourceVersion: "24540" uid: 1ee06b31-dd82-44a1-9408-fc5e4481ec46 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:36:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-60a5c namespace: kserve-ci-e2e-test resourceVersion: "25237" uid: 7f7aa29e-9984-46e1-8cae-fe52c987f531 kind: Event lastTimestamp: "2026-04-20T20:36:08Z" message: InferenceService [success-200-isvc-60a5c] is Ready metadata: creationTimestamp: "2026-04-20T20:36:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:36:08Z" name: success-200-isvc-60a5c.18a82b08685b2229 namespace: kserve-ci-e2e-test resourceVersion: "25238" uid: 31b355c5-6b36-4b76-87b2-d8da758b56f3 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-20T20:24:55.157155Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-63de8-predictor-7d6b9c5fd-p9j7s namespace: kserve-ci-e2e-test resourceVersion: "19308" uid: 967bd600-f4c2-4001-97a3-77b3ce9d9c1c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-63de8-predictor-7d6b9c5fd-p9j7s to ip-10-0-140-19.ec2.internal metadata: creationTimestamp: "2026-04-20T20:24:55Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-20T20:24:55Z" name: success-200-isvc-63de8-predictor-7d6b9c5fd-p9j7s.18a82a6bb9593bb9 namespace: kserve-ci-e2e-test resourceVersion: "19313" uid: 518c2e1f-4c11-49a7-9ba0-e8fd9a5977e3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58dbb994fc-hbcnb source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:24:55Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-63de8-predictor-7d6b9c5fd-p9j7s namespace: kserve-ci-e2e-test resourceVersion: "19311" uid: 967bd600-f4c2-4001-97a3-77b3ce9d9c1c kind: Event lastTimestamp: "2026-04-20T20:24:55Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-63de8-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-20T20:24:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:24:55Z" name: success-200-isvc-63de8-predictor-7d6b9c5fd-p9j7s.18a82a6bc0a6b487 namespace: kserve-ci-e2e-test resourceVersion: "19359" uid: 40a87933-e295-4eb9-877a-af640ee84265 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:24:57Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-63de8-predictor-7d6b9c5fd-p9j7s namespace: kserve-ci-e2e-test resourceVersion: "19319" uid: 967bd600-f4c2-4001-97a3-77b3ce9d9c1c kind: Event lastTimestamp: "2026-04-20T20:24:57Z" message: Add eth0 [10.132.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-20T20:24:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-20T20:24:57Z" name: success-200-isvc-63de8-predictor-7d6b9c5fd-p9j7s.18a82a6c2b99db76 namespace: kserve-ci-e2e-test resourceVersion: "19380" uid: 2d375274-82aa-43aa-9a4e-12cc77068eab reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:24:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-63de8-predictor-7d6b9c5fd-p9j7s namespace: kserve-ci-e2e-test resourceVersion: "19311" uid: 967bd600-f4c2-4001-97a3-77b3ce9d9c1c kind: Event lastTimestamp: "2026-04-20T20:24:57Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-20T20:24:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:24:57Z" name: success-200-isvc-63de8-predictor-7d6b9c5fd-p9j7s.18a82a6c2cd45ce5 namespace: kserve-ci-e2e-test resourceVersion: "19382" uid: f84f99ad-318b-4da3-a5a3-b7f92a210b3a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:24:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-63de8-predictor-7d6b9c5fd-p9j7s namespace: kserve-ci-e2e-test resourceVersion: "19311" uid: 967bd600-f4c2-4001-97a3-77b3ce9d9c1c kind: Event lastTimestamp: "2026-04-20T20:24:57Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-20T20:24:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:24:57Z" name: success-200-isvc-63de8-predictor-7d6b9c5fd-p9j7s.18a82a6c2fa2609d namespace: kserve-ci-e2e-test resourceVersion: "19384" uid: ab9fb2dd-cb6b-4080-af1c-a4a16e419b23 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:24:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-63de8-predictor-7d6b9c5fd-p9j7s namespace: kserve-ci-e2e-test resourceVersion: "19311" uid: 967bd600-f4c2-4001-97a3-77b3ce9d9c1c kind: Event lastTimestamp: "2026-04-20T20:24:57Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-20T20:24:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:24:57Z" name: success-200-isvc-63de8-predictor-7d6b9c5fd-p9j7s.18a82a6c300d1034 namespace: kserve-ci-e2e-test resourceVersion: "19385" uid: 0c977215-5aef-49c1-8cec-e005622e3e4d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:24:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-63de8-predictor-7d6b9c5fd-p9j7s namespace: kserve-ci-e2e-test resourceVersion: "19311" uid: 967bd600-f4c2-4001-97a3-77b3ce9d9c1c kind: Event lastTimestamp: "2026-04-20T20:24:57Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-20T20:24:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:24:57Z" name: success-200-isvc-63de8-predictor-7d6b9c5fd-p9j7s.18a82a6c3017d6de namespace: kserve-ci-e2e-test resourceVersion: "19386" uid: 381b6995-3e17-45f2-a5d3-e6f963fa42a0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:24:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-63de8-predictor-7d6b9c5fd-p9j7s namespace: kserve-ci-e2e-test resourceVersion: "19311" uid: 967bd600-f4c2-4001-97a3-77b3ce9d9c1c kind: Event lastTimestamp: "2026-04-20T20:24:57Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-20T20:24:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:24:57Z" name: success-200-isvc-63de8-predictor-7d6b9c5fd-p9j7s.18a82a6c3281a510 namespace: kserve-ci-e2e-test resourceVersion: "19387" uid: b2fdf6fa-0ac0-4288-8431-e73748a1b5b6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:24:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-63de8-predictor-7d6b9c5fd-p9j7s namespace: kserve-ci-e2e-test resourceVersion: "19311" uid: 967bd600-f4c2-4001-97a3-77b3ce9d9c1c kind: Event lastTimestamp: "2026-04-20T20:24:57Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:24:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:24:57Z" name: success-200-isvc-63de8-predictor-7d6b9c5fd-p9j7s.18a82a6c32efb3f5 namespace: kserve-ci-e2e-test resourceVersion: "19388" uid: 5dfe790d-8fee-4a5b-81f5-425a7ae798c3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-20T20:24:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-63de8-predictor-7d6b9c5fd-p9j7s namespace: kserve-ci-e2e-test resourceVersion: "19311" uid: 967bd600-f4c2-4001-97a3-77b3ce9d9c1c kind: Event lastTimestamp: "2026-04-20T20:25:34Z" message: 'Readiness probe failed: dial tcp 10.132.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-20T20:24:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:25:34Z" name: success-200-isvc-63de8-predictor-7d6b9c5fd-p9j7s.18a82a6c718340e2 namespace: kserve-ci-e2e-test resourceVersion: "19899" uid: 35a70b45-1ecb-4e12-a457-4ecba4716771 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:26:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-63de8-predictor-7d6b9c5fd-p9j7s namespace: kserve-ci-e2e-test resourceVersion: "19311" uid: 967bd600-f4c2-4001-97a3-77b3ce9d9c1c kind: Event lastTimestamp: "2026-04-20T20:26:05Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-20T20:26:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:26:05Z" name: success-200-isvc-63de8-predictor-7d6b9c5fd-p9j7s.18a82a7c12e79208 namespace: kserve-ci-e2e-test resourceVersion: "20262" uid: e2066017-e1dd-46c5-b4be-70f633ab48c9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:26:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-63de8-predictor-7d6b9c5fd-p9j7s namespace: kserve-ci-e2e-test resourceVersion: "19311" uid: 967bd600-f4c2-4001-97a3-77b3ce9d9c1c kind: Event lastTimestamp: "2026-04-20T20:26:05Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:26:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:26:05Z" name: success-200-isvc-63de8-predictor-7d6b9c5fd-p9j7s.18a82a7c12e81411 namespace: kserve-ci-e2e-test resourceVersion: "20264" uid: 8c2311a0-a68b-4252-bb6a-eb7638a40848 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:24:55Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-63de8-predictor-7d6b9c5fd namespace: kserve-ci-e2e-test resourceVersion: "19297" uid: 352a4b8c-b5d4-49f2-9b82-454ccd9348ab kind: Event lastTimestamp: "2026-04-20T20:24:55Z" message: 'Created pod: success-200-isvc-63de8-predictor-7d6b9c5fd-p9j7s' metadata: creationTimestamp: "2026-04-20T20:24:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:24:55Z" name: success-200-isvc-63de8-predictor-7d6b9c5fd.18a82a6bb8e4c297 namespace: kserve-ci-e2e-test resourceVersion: "19310" uid: bd5aaa6f-1f92-49ae-ba91-89ddf4187cbe reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:24:55Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-63de8-predictor namespace: kserve-ci-e2e-test resourceVersion: "19291" uid: f2fc4410-cef0-441c-922a-3348a27e9c89 kind: Event lastTimestamp: "2026-04-20T20:24:55Z" message: Scaled up replica set success-200-isvc-63de8-predictor-7d6b9c5fd from 0 to 1 metadata: creationTimestamp: "2026-04-20T20:24:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:24:55Z" name: success-200-isvc-63de8-predictor.18a82a6bb71640d3 namespace: kserve-ci-e2e-test resourceVersion: "19299" uid: e44d7a3e-35f7-48c9-b38c-6938e44c58ba reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:25:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-63de8-predictor namespace: kserve-ci-e2e-test resourceVersion: "19296" uid: 9051a584-4f67-440f-98f4-0414d912b504 kind: Event lastTimestamp: "2026-04-20T20:25:25Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:25:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:25:25Z" name: success-200-isvc-63de8-predictor.18a82a6f3582ca2b namespace: kserve-ci-e2e-test resourceVersion: "19686" uid: 3be16277-e546-4f40-8b5a-032b908a704b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:25:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-63de8-predictor namespace: kserve-ci-e2e-test resourceVersion: "19296" uid: 9051a584-4f67-440f-98f4-0414d912b504 kind: Event lastTimestamp: "2026-04-20T20:25:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:25:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:25:25Z" name: success-200-isvc-63de8-predictor.18a82a6f3634151e namespace: kserve-ci-e2e-test resourceVersion: "19687" uid: 0fd3fc40-de33-45c6-a834-9d2f684a322f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:25:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-63de8-predictor namespace: kserve-ci-e2e-test resourceVersion: "19519" uid: 9051a584-4f67-440f-98f4-0414d912b504 kind: Event lastTimestamp: "2026-04-20T20:25:55Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:25:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:25:55Z" name: success-200-isvc-63de8-predictor.18a82a7633b8df6c namespace: kserve-ci-e2e-test resourceVersion: "20102" uid: 9952cacf-50d8-4f0a-bbd7-be4ba7b48317 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:25:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-63de8-predictor namespace: kserve-ci-e2e-test resourceVersion: "19519" uid: 9051a584-4f67-440f-98f4-0414d912b504 kind: Event lastTimestamp: "2026-04-20T20:25:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:25:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:25:55Z" name: success-200-isvc-63de8-predictor.18a82a76341f1b22 namespace: kserve-ci-e2e-test resourceVersion: "20104" uid: e2f89187-1da3-4170-bf4c-c9d85a67d903 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-20T20:24:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-63de8 namespace: kserve-ci-e2e-test resourceVersion: "19266" uid: a8155643-1322-4219-92d5-d207c570e848 kind: Event lastTimestamp: "2026-04-20T20:25:44Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-20T20:24:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:25:44Z" name: success-200-isvc-63de8.18a82a6bb6ea5844 namespace: kserve-ci-e2e-test resourceVersion: "20008" uid: 45b47ed9-6305-41ff-b1c4-9bd0d8072f35 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:24:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-63de8 namespace: kserve-ci-e2e-test resourceVersion: "19266" uid: a8155643-1322-4219-92d5-d207c570e848 kind: Event lastTimestamp: "2026-04-20T20:24:55Z" message: 'Failed to update status for InferenceService "success-200-isvc-63de8": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-63de8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-20T20:24:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:24:55Z" name: success-200-isvc-63de8.18a82a6bb76081cd namespace: kserve-ci-e2e-test resourceVersion: "19303" uid: c59c2e7d-cc30-4d38-9132-912dc7f7a94f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:24:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-63de8 namespace: kserve-ci-e2e-test resourceVersion: "19266" uid: a8155643-1322-4219-92d5-d207c570e848 kind: Event lastTimestamp: "2026-04-20T20:24:55Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-63de8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-20T20:24:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:24:55Z" name: success-200-isvc-63de8.18a82a6bb760bf99 namespace: kserve-ci-e2e-test resourceVersion: "19304" uid: ef5c4049-4d84-4ae2-be84-74a40e545940 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-63de8 namespace: kserve-ci-e2e-test resourceVersion: "20005" uid: a8155643-1322-4219-92d5-d207c570e848 kind: Event lastTimestamp: "2026-04-20T20:25:44Z" message: InferenceService [success-200-isvc-63de8] is Ready metadata: creationTimestamp: "2026-04-20T20:25:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:25:44Z" name: success-200-isvc-63de8.18a82a772be33042 namespace: kserve-ci-e2e-test resourceVersion: "20006" uid: 90e4fc3f-959d-425e-8e3e-1a0a0e06d6a2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-20T20:24:15.388488Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7541f-predictor-85fdf68876-fwrsn namespace: kserve-ci-e2e-test resourceVersion: "18861" uid: 9596c6c3-0a23-4d76-9f24-b392c23700e0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-7541f-predictor-85fdf68876-fwrsn to ip-10-0-140-19.ec2.internal metadata: creationTimestamp: "2026-04-20T20:24:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-20T20:24:15Z" name: success-200-isvc-7541f-predictor-85fdf68876-fwrsn.18a82a6276f39602 namespace: kserve-ci-e2e-test resourceVersion: "18867" uid: 52dcce6f-64ad-43e5-b337-024d0ddb1768 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58dbb994fc-hbcnb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:24:15Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7541f-predictor-85fdf68876-fwrsn namespace: kserve-ci-e2e-test resourceVersion: "18863" uid: 9596c6c3-0a23-4d76-9f24-b392c23700e0 kind: Event lastTimestamp: "2026-04-20T20:24:15Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-7541f-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-20T20:24:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:24:15Z" name: success-200-isvc-7541f-predictor-85fdf68876-fwrsn.18a82a62848eefcc namespace: kserve-ci-e2e-test resourceVersion: "18899" uid: 41557500-01b7-4678-87e5-cd0cd059c9d7 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:24:16Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7541f-predictor-85fdf68876-fwrsn namespace: kserve-ci-e2e-test resourceVersion: "18877" uid: 9596c6c3-0a23-4d76-9f24-b392c23700e0 kind: Event lastTimestamp: "2026-04-20T20:24:16Z" message: Add eth0 [10.132.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-20T20:24:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-20T20:24:16Z" name: success-200-isvc-7541f-predictor-85fdf68876-fwrsn.18a82a62b376afa7 namespace: kserve-ci-e2e-test resourceVersion: "18919" uid: dd51b804-5ec9-4598-8f09-32c4648682fe reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:24:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7541f-predictor-85fdf68876-fwrsn namespace: kserve-ci-e2e-test resourceVersion: "18863" uid: 9596c6c3-0a23-4d76-9f24-b392c23700e0 kind: Event lastTimestamp: "2026-04-20T20:24:16Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-20T20:24:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:24:16Z" name: success-200-isvc-7541f-predictor-85fdf68876-fwrsn.18a82a62b4b6f8c9 namespace: kserve-ci-e2e-test resourceVersion: "18922" uid: 01865467-c306-4609-886c-1672f418e7f7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:24:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7541f-predictor-85fdf68876-fwrsn namespace: kserve-ci-e2e-test resourceVersion: "18863" uid: 9596c6c3-0a23-4d76-9f24-b392c23700e0 kind: Event lastTimestamp: "2026-04-20T20:24:16Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-20T20:24:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:24:16Z" name: success-200-isvc-7541f-predictor-85fdf68876-fwrsn.18a82a62b8d0347f namespace: kserve-ci-e2e-test resourceVersion: "18925" uid: a50cdf96-e40f-4445-a45f-72596e5e932b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:24:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7541f-predictor-85fdf68876-fwrsn namespace: kserve-ci-e2e-test resourceVersion: "18863" uid: 9596c6c3-0a23-4d76-9f24-b392c23700e0 kind: Event lastTimestamp: "2026-04-20T20:24:16Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-20T20:24:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:24:16Z" name: success-200-isvc-7541f-predictor-85fdf68876-fwrsn.18a82a62b93cf4bf namespace: kserve-ci-e2e-test resourceVersion: "18926" uid: 027be039-5d09-4310-a11f-977bcc082344 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:24:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7541f-predictor-85fdf68876-fwrsn namespace: kserve-ci-e2e-test resourceVersion: "18863" uid: 9596c6c3-0a23-4d76-9f24-b392c23700e0 kind: Event lastTimestamp: "2026-04-20T20:24:16Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-20T20:24:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:24:16Z" name: success-200-isvc-7541f-predictor-85fdf68876-fwrsn.18a82a62b94794c6 namespace: kserve-ci-e2e-test resourceVersion: "18927" uid: 647d4828-2fbd-4179-ad4d-89f8ad81c2d4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:24:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7541f-predictor-85fdf68876-fwrsn namespace: kserve-ci-e2e-test resourceVersion: "18863" uid: 9596c6c3-0a23-4d76-9f24-b392c23700e0 kind: Event lastTimestamp: "2026-04-20T20:24:16Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-20T20:24:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:24:16Z" name: success-200-isvc-7541f-predictor-85fdf68876-fwrsn.18a82a62bbcf1c6c namespace: kserve-ci-e2e-test resourceVersion: "18928" uid: 4221c02e-168f-417e-b737-a7d30f46cd59 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:24:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7541f-predictor-85fdf68876-fwrsn namespace: kserve-ci-e2e-test resourceVersion: "18863" uid: 9596c6c3-0a23-4d76-9f24-b392c23700e0 kind: Event lastTimestamp: "2026-04-20T20:24:16Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:24:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:24:16Z" name: success-200-isvc-7541f-predictor-85fdf68876-fwrsn.18a82a62bc3d6295 namespace: kserve-ci-e2e-test resourceVersion: "18929" uid: 28f2b5de-e62c-477d-9bbf-2256380bcf78 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-20T20:24:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7541f-predictor-85fdf68876-fwrsn namespace: kserve-ci-e2e-test resourceVersion: "18863" uid: 9596c6c3-0a23-4d76-9f24-b392c23700e0 kind: Event lastTimestamp: "2026-04-20T20:24:54Z" message: 'Readiness probe failed: dial tcp 10.132.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-20T20:24:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:24:54Z" name: success-200-isvc-7541f-predictor-85fdf68876-fwrsn.18a82a631a43de43 namespace: kserve-ci-e2e-test resourceVersion: "19233" uid: 0390d31c-fcf5-4113-8e80-2f57b3f0d0db reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7541f-predictor-85fdf68876-fwrsn namespace: kserve-ci-e2e-test resourceVersion: "18863" uid: 9596c6c3-0a23-4d76-9f24-b392c23700e0 kind: Event lastTimestamp: "2026-04-20T20:25:25Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-20T20:25:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:25:25Z" name: success-200-isvc-7541f-predictor-85fdf68876-fwrsn.18a82a72d27e1a2c namespace: kserve-ci-e2e-test resourceVersion: "19743" uid: 6aa696e7-462e-4e00-8906-06d29f9bca04 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7541f-predictor-85fdf68876-fwrsn namespace: kserve-ci-e2e-test resourceVersion: "18863" uid: 9596c6c3-0a23-4d76-9f24-b392c23700e0 kind: Event lastTimestamp: "2026-04-20T20:25:25Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:25:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:25:25Z" name: success-200-isvc-7541f-predictor-85fdf68876-fwrsn.18a82a72d27eb2f2 namespace: kserve-ci-e2e-test resourceVersion: "19745" uid: 3f9df8ac-32fb-4d10-8304-0c56ac84be77 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:24:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-7541f-predictor-85fdf68876 namespace: kserve-ci-e2e-test resourceVersion: "18842" uid: 87b8a130-92a2-4f72-b1ff-cecf2a68c2de kind: Event lastTimestamp: "2026-04-20T20:24:15Z" message: 'Created pod: success-200-isvc-7541f-predictor-85fdf68876-fwrsn' metadata: creationTimestamp: "2026-04-20T20:24:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:24:15Z" name: success-200-isvc-7541f-predictor-85fdf68876.18a82a62768f38e9 namespace: kserve-ci-e2e-test resourceVersion: "18862" uid: 944e25b7-b438-4281-abcf-6ea1c8fc7bec reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:24:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-7541f-predictor namespace: kserve-ci-e2e-test resourceVersion: "18836" uid: e604c1d1-4555-494b-9d8c-80ca0e042396 kind: Event lastTimestamp: "2026-04-20T20:24:15Z" message: Scaled up replica set success-200-isvc-7541f-predictor-85fdf68876 from 0 to 1 metadata: creationTimestamp: "2026-04-20T20:24:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:24:15Z" name: success-200-isvc-7541f-predictor.18a82a62730aea6b namespace: kserve-ci-e2e-test resourceVersion: "18845" uid: bf812f4f-83ec-4094-8056-cbaa8f5dd1ef reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:24:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7541f-predictor namespace: kserve-ci-e2e-test resourceVersion: "18841" uid: 12303ba0-c7a0-4cda-8cbe-45033c042d09 kind: Event lastTimestamp: "2026-04-20T20:24:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:24:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:24:45Z" name: success-200-isvc-7541f-predictor.18a82a65f1ac7d87 namespace: kserve-ci-e2e-test resourceVersion: "19164" uid: 50327149-ac67-44c6-b5a7-8e4886c3b7cf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:24:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7541f-predictor namespace: kserve-ci-e2e-test resourceVersion: "18841" uid: 12303ba0-c7a0-4cda-8cbe-45033c042d09 kind: Event lastTimestamp: "2026-04-20T20:24:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:24:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:24:45Z" name: success-200-isvc-7541f-predictor.18a82a65f208482c namespace: kserve-ci-e2e-test resourceVersion: "19165" uid: b264a78f-d831-4d99-8012-3e4ba98fe1a0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:25:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7541f-predictor namespace: kserve-ci-e2e-test resourceVersion: "19054" uid: 12303ba0-c7a0-4cda-8cbe-45033c042d09 kind: Event lastTimestamp: "2026-04-20T20:25:15Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:25:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:25:15Z" name: success-200-isvc-7541f-predictor.18a82a6cefbe9e5e namespace: kserve-ci-e2e-test resourceVersion: "19571" uid: a92834e8-b590-46b5-9f57-1c1d1d5fc6af reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:25:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7541f-predictor namespace: kserve-ci-e2e-test resourceVersion: "19054" uid: 12303ba0-c7a0-4cda-8cbe-45033c042d09 kind: Event lastTimestamp: "2026-04-20T20:25:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:25:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:25:15Z" name: success-200-isvc-7541f-predictor.18a82a6cf01c8fef namespace: kserve-ci-e2e-test resourceVersion: "19572" uid: c41528c5-5636-4700-a7a1-1e4848365cd8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-20T20:24:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7541f namespace: kserve-ci-e2e-test resourceVersion: "18812" uid: 27209bf4-3182-4059-bea1-c93fcb9c15c6 kind: Event lastTimestamp: "2026-04-20T20:25:04Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-20T20:24:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:25:04Z" name: success-200-isvc-7541f.18a82a627312ef5c namespace: kserve-ci-e2e-test resourceVersion: "19478" uid: ccd0dc8b-f95f-476f-be38-dc015b0e14e3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:24:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7541f namespace: kserve-ci-e2e-test resourceVersion: "18812" uid: 27209bf4-3182-4059-bea1-c93fcb9c15c6 kind: Event lastTimestamp: "2026-04-20T20:24:15Z" message: 'Failed to update status for InferenceService "success-200-isvc-7541f": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-7541f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-20T20:24:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:24:15Z" name: success-200-isvc-7541f.18a82a6273b10331 namespace: kserve-ci-e2e-test resourceVersion: "18853" uid: a6ba7e5f-ae1b-4b29-b91b-aa60019f1e04 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:24:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7541f namespace: kserve-ci-e2e-test resourceVersion: "18812" uid: 27209bf4-3182-4059-bea1-c93fcb9c15c6 kind: Event lastTimestamp: "2026-04-20T20:24:15Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-7541f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-20T20:24:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:24:15Z" name: success-200-isvc-7541f.18a82a6273b13bb7 namespace: kserve-ci-e2e-test resourceVersion: "18855" uid: 890293e3-e904-4c4a-b444-97e8dd6da8a5 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7541f namespace: kserve-ci-e2e-test resourceVersion: "19475" uid: 27209bf4-3182-4059-bea1-c93fcb9c15c6 kind: Event lastTimestamp: "2026-04-20T20:25:04Z" message: InferenceService [success-200-isvc-7541f] is Ready metadata: creationTimestamp: "2026-04-20T20:25:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:25:04Z" name: success-200-isvc-7541f.18a82a6dd4be538e namespace: kserve-ci-e2e-test resourceVersion: "19476" uid: f1816664-cce5-4e99-a15e-8cec48bf32be reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-20T20:15:00.337191Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-9290e-predictor-5b44755bcb-smm7n namespace: kserve-ci-e2e-test resourceVersion: "14455" uid: 79498564-cc7d-4361-9213-90d899860d1e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-9290e-predictor-5b44755bcb-smm7n to ip-10-0-140-19.ec2.internal metadata: creationTimestamp: "2026-04-20T20:15:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-20T20:15:00Z" name: success-200-isvc-9290e-predictor-5b44755bcb-smm7n.18a829e13b50e665 namespace: kserve-ci-e2e-test resourceVersion: "14460" uid: 1ac03096-0977-42b1-9897-c9019bd6a440 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58dbb994fc-hbcnb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:00Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-9290e-predictor-5b44755bcb-smm7n namespace: kserve-ci-e2e-test resourceVersion: "14458" uid: 79498564-cc7d-4361-9213-90d899860d1e kind: Event lastTimestamp: "2026-04-20T20:15:00Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-9290e-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-20T20:15:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:15:00Z" name: success-200-isvc-9290e-predictor-5b44755bcb-smm7n.18a829e14b72b55c namespace: kserve-ci-e2e-test resourceVersion: "14492" uid: f916f2fa-f84f-4678-a197-3c80cc39bf5f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:01Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-9290e-predictor-5b44755bcb-smm7n namespace: kserve-ci-e2e-test resourceVersion: "14464" uid: 79498564-cc7d-4361-9213-90d899860d1e kind: Event lastTimestamp: "2026-04-20T20:15:01Z" message: Add eth0 [10.132.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-20T20:15:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-20T20:15:01Z" name: success-200-isvc-9290e-predictor-5b44755bcb-smm7n.18a829e177c4d5ba namespace: kserve-ci-e2e-test resourceVersion: "14515" uid: cf7c7241-950d-4051-bda6-10cc24a8e01d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9290e-predictor-5b44755bcb-smm7n namespace: kserve-ci-e2e-test resourceVersion: "14458" uid: 79498564-cc7d-4361-9213-90d899860d1e kind: Event lastTimestamp: "2026-04-20T20:15:01Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-20T20:15:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:15:01Z" name: success-200-isvc-9290e-predictor-5b44755bcb-smm7n.18a829e1790bd258 namespace: kserve-ci-e2e-test resourceVersion: "14517" uid: 2d81d14a-bc5f-4401-af62-80813796a3c7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9290e-predictor-5b44755bcb-smm7n namespace: kserve-ci-e2e-test resourceVersion: "14458" uid: 79498564-cc7d-4361-9213-90d899860d1e kind: Event lastTimestamp: "2026-04-20T20:15:01Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-20T20:15:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:15:01Z" name: success-200-isvc-9290e-predictor-5b44755bcb-smm7n.18a829e17bc4383a namespace: kserve-ci-e2e-test resourceVersion: "14519" uid: b0d6227a-d653-4171-accd-fdfdf76340a2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9290e-predictor-5b44755bcb-smm7n namespace: kserve-ci-e2e-test resourceVersion: "14458" uid: 79498564-cc7d-4361-9213-90d899860d1e kind: Event lastTimestamp: "2026-04-20T20:15:01Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-20T20:15:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:15:01Z" name: success-200-isvc-9290e-predictor-5b44755bcb-smm7n.18a829e17c453efe namespace: kserve-ci-e2e-test resourceVersion: "14520" uid: 527bb862-b5a1-4101-b9b3-e3435d940bcc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9290e-predictor-5b44755bcb-smm7n namespace: kserve-ci-e2e-test resourceVersion: "14458" uid: 79498564-cc7d-4361-9213-90d899860d1e kind: Event lastTimestamp: "2026-04-20T20:15:01Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-20T20:15:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:15:01Z" name: success-200-isvc-9290e-predictor-5b44755bcb-smm7n.18a829e17c51074a namespace: kserve-ci-e2e-test resourceVersion: "14521" uid: 9e83200d-16d5-4428-9988-02d3b35c7ede reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9290e-predictor-5b44755bcb-smm7n namespace: kserve-ci-e2e-test resourceVersion: "14458" uid: 79498564-cc7d-4361-9213-90d899860d1e kind: Event lastTimestamp: "2026-04-20T20:15:01Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-20T20:15:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:15:01Z" name: success-200-isvc-9290e-predictor-5b44755bcb-smm7n.18a829e17f1f11ff namespace: kserve-ci-e2e-test resourceVersion: "14522" uid: e12b15b7-a6af-4ab1-9647-c7651bea0dd7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9290e-predictor-5b44755bcb-smm7n namespace: kserve-ci-e2e-test resourceVersion: "14458" uid: 79498564-cc7d-4361-9213-90d899860d1e kind: Event lastTimestamp: "2026-04-20T20:15:01Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:15:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:15:01Z" name: success-200-isvc-9290e-predictor-5b44755bcb-smm7n.18a829e17f8b4b41 namespace: kserve-ci-e2e-test resourceVersion: "14523" uid: 3db1e0c6-92bf-4007-8429-b6336e8ca980 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-20T20:15:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9290e-predictor-5b44755bcb-smm7n namespace: kserve-ci-e2e-test resourceVersion: "14458" uid: 79498564-cc7d-4361-9213-90d899860d1e kind: Event lastTimestamp: "2026-04-20T20:15:38Z" message: 'Readiness probe failed: dial tcp 10.132.0.20:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-20T20:15:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:15:38Z" name: success-200-isvc-9290e-predictor-5b44755bcb-smm7n.18a829e1bea8ab73 namespace: kserve-ci-e2e-test resourceVersion: "14903" uid: 1c51d057-964a-4a5c-8996-c1474db428e0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:24:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9290e-predictor-5b44755bcb-smm7n namespace: kserve-ci-e2e-test resourceVersion: "14458" uid: 79498564-cc7d-4361-9213-90d899860d1e kind: Event lastTimestamp: "2026-04-20T20:24:15Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-20T20:24:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:24:15Z" name: success-200-isvc-9290e-predictor-5b44755bcb-smm7n.18a82a6271cc8e3c namespace: kserve-ci-e2e-test resourceVersion: "18831" uid: 2ff9275b-3188-4d40-878d-9951f255337f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:24:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9290e-predictor-5b44755bcb-smm7n namespace: kserve-ci-e2e-test resourceVersion: "14458" uid: 79498564-cc7d-4361-9213-90d899860d1e kind: Event lastTimestamp: "2026-04-20T20:24:15Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:24:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:24:15Z" name: success-200-isvc-9290e-predictor-5b44755bcb-smm7n.18a82a6271cd1c10 namespace: kserve-ci-e2e-test resourceVersion: "18835" uid: b3cfb115-2ffe-4a91-b7fd-50b3697622b5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-9290e-predictor-5b44755bcb namespace: kserve-ci-e2e-test resourceVersion: "14445" uid: 50ddc85b-84de-41ab-89ba-1c02ad41a36b kind: Event lastTimestamp: "2026-04-20T20:15:00Z" message: 'Created pod: success-200-isvc-9290e-predictor-5b44755bcb-smm7n' metadata: creationTimestamp: "2026-04-20T20:15:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:15:00Z" name: success-200-isvc-9290e-predictor-5b44755bcb.18a829e13adf8217 namespace: kserve-ci-e2e-test resourceVersion: "14457" uid: bdf6db37-b65a-4f64-8aef-2747a6059143 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-9290e-predictor namespace: kserve-ci-e2e-test resourceVersion: "14441" uid: bea56f66-3c29-47d9-96b2-4084c712ae02 kind: Event lastTimestamp: "2026-04-20T20:15:00Z" message: Scaled up replica set success-200-isvc-9290e-predictor-5b44755bcb from 0 to 1 metadata: creationTimestamp: "2026-04-20T20:15:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:15:00Z" name: success-200-isvc-9290e-predictor.18a829e1393f9902 namespace: kserve-ci-e2e-test resourceVersion: "14449" uid: f482415d-d784-482f-972e-2cb8d7bf674b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:15:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9290e-predictor namespace: kserve-ci-e2e-test resourceVersion: "14444" uid: d428650e-5837-4065-86db-78c8c0309b3f kind: Event lastTimestamp: "2026-04-20T20:15:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:15:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:15:30Z" name: success-200-isvc-9290e-predictor.18a829e4b7a12e30 namespace: kserve-ci-e2e-test resourceVersion: "14821" uid: 665ad21b-d5ff-42fd-b4d1-1bdb750c6608 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:15:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9290e-predictor namespace: kserve-ci-e2e-test resourceVersion: "14444" uid: d428650e-5837-4065-86db-78c8c0309b3f kind: Event lastTimestamp: "2026-04-20T20:15:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:15:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:15:30Z" name: success-200-isvc-9290e-predictor.18a829e4b7f308f3 namespace: kserve-ci-e2e-test resourceVersion: "14822" uid: 3206d87f-5c46-47ca-9a84-345fb2f8e4ac reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-20T20:15:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9290e-predictor namespace: kserve-ci-e2e-test resourceVersion: "14673" uid: d428650e-5837-4065-86db-78c8c0309b3f kind: Event lastTimestamp: "2026-04-20T20:16:15Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:15:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:16:15Z" name: success-200-isvc-9290e-predictor.18a829ebb5e41b38 namespace: kserve-ci-e2e-test resourceVersion: "15427" uid: e97d70d9-0c1c-4315-aaef-2c7313d5e4ca reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-20T20:15:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9290e-predictor namespace: kserve-ci-e2e-test resourceVersion: "14673" uid: d428650e-5837-4065-86db-78c8c0309b3f kind: Event lastTimestamp: "2026-04-20T20:16:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:15:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:16:15Z" name: success-200-isvc-9290e-predictor.18a829ebb64c9843 namespace: kserve-ci-e2e-test resourceVersion: "15428" uid: ff788ec2-8365-4511-b978-a34dae7c9685 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-20T20:15:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9290e namespace: kserve-ci-e2e-test resourceVersion: "14412" uid: 0429b19c-99a2-4510-b419-78ec010a8225 kind: Event lastTimestamp: "2026-04-20T20:15:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-20T20:15:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:15:48Z" name: success-200-isvc-9290e.18a829e1390bb904 namespace: kserve-ci-e2e-test resourceVersion: "15175" uid: 305766ad-8bd1-46d0-bba4-b4ce6f7c0814 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9290e namespace: kserve-ci-e2e-test resourceVersion: "14412" uid: 0429b19c-99a2-4510-b419-78ec010a8225 kind: Event lastTimestamp: "2026-04-20T20:15:00Z" message: 'Failed to update status for InferenceService "success-200-isvc-9290e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-9290e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-20T20:15:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:15:00Z" name: success-200-isvc-9290e.18a829e139a35cdf namespace: kserve-ci-e2e-test resourceVersion: "14451" uid: fc16bc7b-f8d3-4f13-b6a4-78f2c5c57291 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9290e namespace: kserve-ci-e2e-test resourceVersion: "14412" uid: 0429b19c-99a2-4510-b419-78ec010a8225 kind: Event lastTimestamp: "2026-04-20T20:15:00Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-9290e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-20T20:15:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:15:00Z" name: success-200-isvc-9290e.18a829e139a3ab74 namespace: kserve-ci-e2e-test resourceVersion: "14453" uid: cea84d17-921d-4502-b285-f3abaac9de52 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9290e namespace: kserve-ci-e2e-test resourceVersion: "15173" uid: 0429b19c-99a2-4510-b419-78ec010a8225 kind: Event lastTimestamp: "2026-04-20T20:15:48Z" message: InferenceService [success-200-isvc-9290e] is Ready metadata: creationTimestamp: "2026-04-20T20:15:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:15:48Z" name: success-200-isvc-9290e.18a829ec796f3f13 namespace: kserve-ci-e2e-test resourceVersion: "15174" uid: dc25a603-b802-40ad-8a17-cf55334a6b89 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-20T20:34:40.539650Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ab6b8-predictor-6857df4dd-4wr8k namespace: kserve-ci-e2e-test resourceVersion: "24111" uid: 2270c01b-2bf1-48db-a32d-6f73b6d0dd32 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-ab6b8-predictor-6857df4dd-4wr8k to ip-10-0-140-19.ec2.internal metadata: creationTimestamp: "2026-04-20T20:34:40Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-20T20:34:40Z" name: success-200-isvc-ab6b8-predictor-6857df4dd-4wr8k.18a82af404dd4ac6 namespace: kserve-ci-e2e-test resourceVersion: "24118" uid: db65ea97-eec3-4e11-acd4-dcd0f442baf3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58dbb994fc-hbcnb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:34:40Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ab6b8-predictor-6857df4dd-4wr8k namespace: kserve-ci-e2e-test resourceVersion: "24133" uid: 2270c01b-2bf1-48db-a32d-6f73b6d0dd32 kind: Event lastTimestamp: "2026-04-20T20:34:40Z" message: Add eth0 [10.132.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-20T20:34:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-20T20:34:40Z" name: success-200-isvc-ab6b8-predictor-6857df4dd-4wr8k.18a82af41dc4a51f namespace: kserve-ci-e2e-test resourceVersion: "24157" uid: acdbee73-c2de-417d-bd34-a977afb3fc32 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:34:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ab6b8-predictor-6857df4dd-4wr8k namespace: kserve-ci-e2e-test resourceVersion: "24115" uid: 2270c01b-2bf1-48db-a32d-6f73b6d0dd32 kind: Event lastTimestamp: "2026-04-20T20:34:40Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-20T20:34:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:34:40Z" name: success-200-isvc-ab6b8-predictor-6857df4dd-4wr8k.18a82af41f0906a9 namespace: kserve-ci-e2e-test resourceVersion: "24159" uid: 751cf19f-ff2d-4d5f-93a5-e906517dc098 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:34:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ab6b8-predictor-6857df4dd-4wr8k namespace: kserve-ci-e2e-test resourceVersion: "24115" uid: 2270c01b-2bf1-48db-a32d-6f73b6d0dd32 kind: Event lastTimestamp: "2026-04-20T20:34:41Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-20T20:34:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:34:41Z" name: success-200-isvc-ab6b8-predictor-6857df4dd-4wr8k.18a82af421f2f24f namespace: kserve-ci-e2e-test resourceVersion: "24160" uid: 0e2170a0-a395-4db1-826f-573a10268b19 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:34:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ab6b8-predictor-6857df4dd-4wr8k namespace: kserve-ci-e2e-test resourceVersion: "24115" uid: 2270c01b-2bf1-48db-a32d-6f73b6d0dd32 kind: Event lastTimestamp: "2026-04-20T20:34:41Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-20T20:34:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:34:41Z" name: success-200-isvc-ab6b8-predictor-6857df4dd-4wr8k.18a82af4226a9c7e namespace: kserve-ci-e2e-test resourceVersion: "24161" uid: e7686b23-3246-4da6-9384-e702f21242fb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:34:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ab6b8-predictor-6857df4dd-4wr8k namespace: kserve-ci-e2e-test resourceVersion: "24115" uid: 2270c01b-2bf1-48db-a32d-6f73b6d0dd32 kind: Event lastTimestamp: "2026-04-20T20:34:41Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-20T20:34:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:34:41Z" name: success-200-isvc-ab6b8-predictor-6857df4dd-4wr8k.18a82af4227606af namespace: kserve-ci-e2e-test resourceVersion: "24162" uid: cf34688c-1d61-40a0-a8ad-35f161a7fe8a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:34:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ab6b8-predictor-6857df4dd-4wr8k namespace: kserve-ci-e2e-test resourceVersion: "24115" uid: 2270c01b-2bf1-48db-a32d-6f73b6d0dd32 kind: Event lastTimestamp: "2026-04-20T20:34:41Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-20T20:34:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:34:41Z" name: success-200-isvc-ab6b8-predictor-6857df4dd-4wr8k.18a82af42476bb35 namespace: kserve-ci-e2e-test resourceVersion: "24163" uid: 7252675a-b962-42fd-a034-27de58cea3dc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:34:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ab6b8-predictor-6857df4dd-4wr8k namespace: kserve-ci-e2e-test resourceVersion: "24115" uid: 2270c01b-2bf1-48db-a32d-6f73b6d0dd32 kind: Event lastTimestamp: "2026-04-20T20:34:41Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:34:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:34:41Z" name: success-200-isvc-ab6b8-predictor-6857df4dd-4wr8k.18a82af424e2849b namespace: kserve-ci-e2e-test resourceVersion: "24164" uid: 89f20f1b-c4d6-48b3-b725-9663ba4b89e7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-20T20:34:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ab6b8-predictor-6857df4dd-4wr8k namespace: kserve-ci-e2e-test resourceVersion: "24115" uid: 2270c01b-2bf1-48db-a32d-6f73b6d0dd32 kind: Event lastTimestamp: "2026-04-20T20:35:17Z" message: 'Readiness probe failed: dial tcp 10.132.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-20T20:34:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:35:17Z" name: success-200-isvc-ab6b8-predictor-6857df4dd-4wr8k.18a82af456c13310 namespace: kserve-ci-e2e-test resourceVersion: "24476" uid: 1e23a65e-3999-41c6-b29b-095cbbbcbd67 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ab6b8-predictor-6857df4dd-4wr8k namespace: kserve-ci-e2e-test resourceVersion: "24115" uid: 2270c01b-2bf1-48db-a32d-6f73b6d0dd32 kind: Event lastTimestamp: "2026-04-20T20:35:50Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-20T20:35:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:35:50Z" name: success-200-isvc-ab6b8-predictor-6857df4dd-4wr8k.18a82b0460970d32 namespace: kserve-ci-e2e-test resourceVersion: "25003" uid: a35982c1-5a65-4c17-9cb9-f48c2d54f51e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ab6b8-predictor-6857df4dd-4wr8k namespace: kserve-ci-e2e-test resourceVersion: "24115" uid: 2270c01b-2bf1-48db-a32d-6f73b6d0dd32 kind: Event lastTimestamp: "2026-04-20T20:35:50Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:35:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:35:50Z" name: success-200-isvc-ab6b8-predictor-6857df4dd-4wr8k.18a82b0460977a2f namespace: kserve-ci-e2e-test resourceVersion: "25004" uid: 729226d3-82fc-41c9-8939-28493a598e27 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ab6b8-predictor-6857df4dd-4wr8k namespace: kserve-ci-e2e-test resourceVersion: "24115" uid: 2270c01b-2bf1-48db-a32d-6f73b6d0dd32 kind: Event lastTimestamp: "2026-04-20T20:35:52Z" message: 'Readiness probe failed: Get "https://10.132.0.26:8643/healthz": dial tcp 10.132.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-20T20:35:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:35:52Z" name: success-200-isvc-ab6b8-predictor-6857df4dd-4wr8k.18a82b04dec11109 namespace: kserve-ci-e2e-test resourceVersion: "25090" uid: 666723eb-a238-45c5-b7f4-7403064d5933 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:34:40Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-ab6b8-predictor-6857df4dd namespace: kserve-ci-e2e-test resourceVersion: "24085" uid: 2052c686-d036-41cc-b169-fd960348d763 kind: Event lastTimestamp: "2026-04-20T20:34:40Z" message: 'Created pod: success-200-isvc-ab6b8-predictor-6857df4dd-4wr8k' metadata: creationTimestamp: "2026-04-20T20:34:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:34:40Z" name: success-200-isvc-ab6b8-predictor-6857df4dd.18a82af4047c3aaf namespace: kserve-ci-e2e-test resourceVersion: "24114" uid: 78c96d83-9ce2-46f2-ab46-d15d1cdb54f3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:34:40Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-ab6b8-predictor namespace: kserve-ci-e2e-test resourceVersion: "24082" uid: fb446e1a-b0a2-4e9b-b14c-1fb050ddadc1 kind: Event lastTimestamp: "2026-04-20T20:34:40Z" message: Scaled up replica set success-200-isvc-ab6b8-predictor-6857df4dd from 0 to 1 metadata: creationTimestamp: "2026-04-20T20:34:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:34:40Z" name: success-200-isvc-ab6b8-predictor.18a82af400462fb5 namespace: kserve-ci-e2e-test resourceVersion: "24086" uid: 8692ecde-88ac-48a3-a2fb-f2cea03a32cd reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:34:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ab6b8-predictor namespace: kserve-ci-e2e-test resourceVersion: "24084" uid: 724547d9-c80e-4b82-9d72-748f0e05173c kind: Event lastTimestamp: "2026-04-20T20:35:10Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:34:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:35:10Z" name: success-200-isvc-ab6b8-predictor.18a82af77edf5b3b namespace: kserve-ci-e2e-test resourceVersion: "24417" uid: f775c099-2d65-498a-ba2d-f600865c21b6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:34:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ab6b8-predictor namespace: kserve-ci-e2e-test resourceVersion: "24084" uid: 724547d9-c80e-4b82-9d72-748f0e05173c kind: Event lastTimestamp: "2026-04-20T20:35:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:34:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:35:10Z" name: success-200-isvc-ab6b8-predictor.18a82af77f417fd0 namespace: kserve-ci-e2e-test resourceVersion: "24418" uid: 7859cda5-3cb9-4ae1-9315-22ea4011b98b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:35:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ab6b8-predictor namespace: kserve-ci-e2e-test resourceVersion: "24305" uid: 724547d9-c80e-4b82-9d72-748f0e05173c kind: Event lastTimestamp: "2026-04-20T20:35:40Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:35:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:35:40Z" name: success-200-isvc-ab6b8-predictor.18a82afe7cb165a0 namespace: kserve-ci-e2e-test resourceVersion: "24816" uid: d8454715-76e9-437d-b4de-658154952952 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:35:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ab6b8-predictor namespace: kserve-ci-e2e-test resourceVersion: "24305" uid: 724547d9-c80e-4b82-9d72-748f0e05173c kind: Event lastTimestamp: "2026-04-20T20:35:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:35:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:35:40Z" name: success-200-isvc-ab6b8-predictor.18a82afe7d18dad5 namespace: kserve-ci-e2e-test resourceVersion: "24817" uid: f8183031-663d-4797-800e-3ad11b156ae6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-20T20:34:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ab6b8 namespace: kserve-ci-e2e-test resourceVersion: "24062" uid: d9f0dfee-7894-4a27-b18b-5975032ce151 kind: Event lastTimestamp: "2026-04-20T20:35:28Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-20T20:34:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:35:28Z" name: success-200-isvc-ab6b8.18a82af4004e6521 namespace: kserve-ci-e2e-test resourceVersion: "24702" uid: 0c376030-572e-44ec-ac0f-4e9f28fe77d8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:34:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ab6b8 namespace: kserve-ci-e2e-test resourceVersion: "24062" uid: d9f0dfee-7894-4a27-b18b-5975032ce151 kind: Event lastTimestamp: "2026-04-20T20:34:40Z" message: 'Failed to update status for InferenceService "success-200-isvc-ab6b8": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-ab6b8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-20T20:34:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:34:40Z" name: success-200-isvc-ab6b8.18a82af400e369b7 namespace: kserve-ci-e2e-test resourceVersion: "24092" uid: c1059c84-ff8e-47b0-9c03-fc2f52e3f61f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:34:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ab6b8 namespace: kserve-ci-e2e-test resourceVersion: "24062" uid: d9f0dfee-7894-4a27-b18b-5975032ce151 kind: Event lastTimestamp: "2026-04-20T20:34:40Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-ab6b8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-20T20:34:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:34:40Z" name: success-200-isvc-ab6b8.18a82af400e3ba31 namespace: kserve-ci-e2e-test resourceVersion: "24094" uid: 8600c742-6ed6-4522-842f-95b764141b61 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ab6b8 namespace: kserve-ci-e2e-test resourceVersion: "24700" uid: d9f0dfee-7894-4a27-b18b-5975032ce151 kind: Event lastTimestamp: "2026-04-20T20:35:27Z" message: InferenceService [success-200-isvc-ab6b8] is Ready metadata: creationTimestamp: "2026-04-20T20:35:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:35:27Z" name: success-200-isvc-ab6b8.18a82aff10dc0567 namespace: kserve-ci-e2e-test resourceVersion: "24701" uid: 98bf066b-ad1f-4475-895c-e31806294663 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-20T20:15:40.490978Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e6f34-predictor-57474bf446-r75x6 namespace: kserve-ci-e2e-test resourceVersion: "14992" uid: c17dfe92-a4ff-4b88-8cfb-aebae882c7fd kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-e6f34-predictor-57474bf446-r75x6 to ip-10-0-140-19.ec2.internal metadata: creationTimestamp: "2026-04-20T20:15:40Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-20T20:15:40Z" name: success-200-isvc-e6f34-predictor-57474bf446-r75x6.18a829ea94ab03f0 namespace: kserve-ci-e2e-test resourceVersion: "15000" uid: 9a4c53e0-9323-46db-9cd6-06c7457e2643 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58dbb994fc-hbcnb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:40Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e6f34-predictor-57474bf446-r75x6 namespace: kserve-ci-e2e-test resourceVersion: "15013" uid: c17dfe92-a4ff-4b88-8cfb-aebae882c7fd kind: Event lastTimestamp: "2026-04-20T20:15:40Z" message: Add eth0 [10.132.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-20T20:15:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-20T20:15:40Z" name: success-200-isvc-e6f34-predictor-57474bf446-r75x6.18a829eaad4e9f4b namespace: kserve-ci-e2e-test resourceVersion: "15056" uid: 24bec2f7-50be-43a9-b753-3c32485ec9d0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e6f34-predictor-57474bf446-r75x6 namespace: kserve-ci-e2e-test resourceVersion: "14998" uid: c17dfe92-a4ff-4b88-8cfb-aebae882c7fd kind: Event lastTimestamp: "2026-04-20T20:15:40Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-20T20:15:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:15:40Z" name: success-200-isvc-e6f34-predictor-57474bf446-r75x6.18a829eaaef9a012 namespace: kserve-ci-e2e-test resourceVersion: "15058" uid: 06482b3f-6146-4721-bd1f-aa4c53a87488 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e6f34-predictor-57474bf446-r75x6 namespace: kserve-ci-e2e-test resourceVersion: "14998" uid: c17dfe92-a4ff-4b88-8cfb-aebae882c7fd kind: Event lastTimestamp: "2026-04-20T20:15:40Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-20T20:15:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:15:40Z" name: success-200-isvc-e6f34-predictor-57474bf446-r75x6.18a829eab1d97563 namespace: kserve-ci-e2e-test resourceVersion: "15059" uid: e33b1812-b856-4404-90d1-cea2c18f8aee reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e6f34-predictor-57474bf446-r75x6 namespace: kserve-ci-e2e-test resourceVersion: "14998" uid: c17dfe92-a4ff-4b88-8cfb-aebae882c7fd kind: Event lastTimestamp: "2026-04-20T20:15:40Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-20T20:15:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:15:40Z" name: success-200-isvc-e6f34-predictor-57474bf446-r75x6.18a829eab2449b4c namespace: kserve-ci-e2e-test resourceVersion: "15060" uid: 7ea48ddb-73d2-4102-8253-200dca0ae107 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e6f34-predictor-57474bf446-r75x6 namespace: kserve-ci-e2e-test resourceVersion: "14998" uid: c17dfe92-a4ff-4b88-8cfb-aebae882c7fd kind: Event lastTimestamp: "2026-04-20T20:15:40Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-20T20:15:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:15:40Z" name: success-200-isvc-e6f34-predictor-57474bf446-r75x6.18a829eab24f5d9b namespace: kserve-ci-e2e-test resourceVersion: "15061" uid: a72d7dd5-771c-4cf8-8371-66a625b9328c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e6f34-predictor-57474bf446-r75x6 namespace: kserve-ci-e2e-test resourceVersion: "14998" uid: c17dfe92-a4ff-4b88-8cfb-aebae882c7fd kind: Event lastTimestamp: "2026-04-20T20:15:41Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-20T20:15:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:15:41Z" name: success-200-isvc-e6f34-predictor-57474bf446-r75x6.18a829eab4884ad9 namespace: kserve-ci-e2e-test resourceVersion: "15063" uid: d3a410d3-b17a-4d05-a5fc-b4bfe40c70b5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e6f34-predictor-57474bf446-r75x6 namespace: kserve-ci-e2e-test resourceVersion: "14998" uid: c17dfe92-a4ff-4b88-8cfb-aebae882c7fd kind: Event lastTimestamp: "2026-04-20T20:15:41Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:15:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:15:41Z" name: success-200-isvc-e6f34-predictor-57474bf446-r75x6.18a829eab50c68c6 namespace: kserve-ci-e2e-test resourceVersion: "15064" uid: c078ea9a-9b13-4651-af65-9a080a589882 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-20T20:15:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e6f34-predictor-57474bf446-r75x6 namespace: kserve-ci-e2e-test resourceVersion: "14998" uid: c17dfe92-a4ff-4b88-8cfb-aebae882c7fd kind: Event lastTimestamp: "2026-04-20T20:16:18Z" message: 'Readiness probe failed: dial tcp 10.132.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-20T20:15:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:16:18Z" name: success-200-isvc-e6f34-predictor-57474bf446-r75x6.18a829eb16a4b99a namespace: kserve-ci-e2e-test resourceVersion: "15455" uid: 885c8cb9-71d4-46c7-a1ee-b9657b3b1618 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:24:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e6f34-predictor-57474bf446-r75x6 namespace: kserve-ci-e2e-test resourceVersion: "14998" uid: c17dfe92-a4ff-4b88-8cfb-aebae882c7fd kind: Event lastTimestamp: "2026-04-20T20:24:55Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-20T20:24:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:24:55Z" name: success-200-isvc-e6f34-predictor-57474bf446-r75x6.18a82a6bb7f5a59d namespace: kserve-ci-e2e-test resourceVersion: "19305" uid: 9cea4cbc-dff1-46ad-add0-0fc3abcea867 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:24:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e6f34-predictor-57474bf446-r75x6 namespace: kserve-ci-e2e-test resourceVersion: "14998" uid: c17dfe92-a4ff-4b88-8cfb-aebae882c7fd kind: Event lastTimestamp: "2026-04-20T20:24:55Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:24:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:24:55Z" name: success-200-isvc-e6f34-predictor-57474bf446-r75x6.18a82a6bb7f69c27 namespace: kserve-ci-e2e-test resourceVersion: "19307" uid: a52192ec-f31f-46a6-ad73-303329433409 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:40Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-e6f34-predictor-57474bf446 namespace: kserve-ci-e2e-test resourceVersion: "14976" uid: ed915152-9852-48e3-b247-a6f5414e8aaa kind: Event lastTimestamp: "2026-04-20T20:15:40Z" message: 'Created pod: success-200-isvc-e6f34-predictor-57474bf446-r75x6' metadata: creationTimestamp: "2026-04-20T20:15:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:15:40Z" name: success-200-isvc-e6f34-predictor-57474bf446.18a829ea93b1ea9d namespace: kserve-ci-e2e-test resourceVersion: "14996" uid: 67ee00ea-a581-401e-83c2-d1506a203701 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:40Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-e6f34-predictor namespace: kserve-ci-e2e-test resourceVersion: "14965" uid: 3837ace2-77db-4c1a-ae3e-104f4af3a5ca kind: Event lastTimestamp: "2026-04-20T20:15:40Z" message: Scaled up replica set success-200-isvc-e6f34-predictor-57474bf446 from 0 to 1 metadata: creationTimestamp: "2026-04-20T20:15:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:15:40Z" name: success-200-isvc-e6f34-predictor.18a829ea8e5e5548 namespace: kserve-ci-e2e-test resourceVersion: "14981" uid: ccb44b13-8d2b-4af1-91c7-b1ea02c04bac reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:15:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e6f34-predictor namespace: kserve-ci-e2e-test resourceVersion: "14968" uid: e15b2636-6a34-4092-88d6-895b697e1c71 kind: Event lastTimestamp: "2026-04-20T20:16:10Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:15:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:16:10Z" name: success-200-isvc-e6f34-predictor.18a829ee0c02cd80 namespace: kserve-ci-e2e-test resourceVersion: "15386" uid: 88017bbd-b763-4882-ba8a-494160274cf3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:15:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e6f34-predictor namespace: kserve-ci-e2e-test resourceVersion: "14968" uid: e15b2636-6a34-4092-88d6-895b697e1c71 kind: Event lastTimestamp: "2026-04-20T20:16:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:15:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:16:10Z" name: success-200-isvc-e6f34-predictor.18a829ee0c7791a3 namespace: kserve-ci-e2e-test resourceVersion: "15387" uid: d7df2011-d353-4a7e-ba45-0a428670ee7e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-20T20:16:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e6f34-predictor namespace: kserve-ci-e2e-test resourceVersion: "15221" uid: e15b2636-6a34-4092-88d6-895b697e1c71 kind: Event lastTimestamp: "2026-04-20T20:17:10Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:16:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:17:10Z" name: success-200-isvc-e6f34-predictor.18a829f509dc0b17 namespace: kserve-ci-e2e-test resourceVersion: "15894" uid: 8192009d-101f-47f8-a229-cd0fcb6f791b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-20T20:16:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e6f34-predictor namespace: kserve-ci-e2e-test resourceVersion: "15221" uid: e15b2636-6a34-4092-88d6-895b697e1c71 kind: Event lastTimestamp: "2026-04-20T20:17:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:16:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:17:10Z" name: success-200-isvc-e6f34-predictor.18a829f50a394d39 namespace: kserve-ci-e2e-test resourceVersion: "15895" uid: e715c8a6-903d-4462-ae65-c13f0e04b1aa reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-20T20:15:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e6f34 namespace: kserve-ci-e2e-test resourceVersion: "14942" uid: d2eabefc-ac16-4621-88f8-7c13adf34cea kind: Event lastTimestamp: "2026-04-20T20:16:28Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-20T20:15:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:16:28Z" name: success-200-isvc-e6f34.18a829ea8d17b613 namespace: kserve-ci-e2e-test resourceVersion: "15543" uid: 5887d467-0616-440c-a5ec-c6006aca1179 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:16:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e6f34 namespace: kserve-ci-e2e-test resourceVersion: "15541" uid: d2eabefc-ac16-4621-88f8-7c13adf34cea kind: Event lastTimestamp: "2026-04-20T20:16:28Z" message: InferenceService [success-200-isvc-e6f34] is Ready metadata: creationTimestamp: "2026-04-20T20:16:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:16:28Z" name: success-200-isvc-e6f34.18a829f5d0fdda0f namespace: kserve-ci-e2e-test resourceVersion: "15542" uid: e8962839-5f35-4690-b829-baad028f1c7a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-20T20:26:05.402035Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-fe410-predictor-66c5568bb-9sm5z namespace: kserve-ci-e2e-test resourceVersion: "20265" uid: 1d11f25d-fee7-4da2-84fa-2de26a1f4cb5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-fe410-predictor-66c5568bb-9sm5z to ip-10-0-140-19.ec2.internal metadata: creationTimestamp: "2026-04-20T20:26:05Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-20T20:26:05Z" name: success-200-isvc-fe410-predictor-66c5568bb-9sm5z.18a82a7c14451301 namespace: kserve-ci-e2e-test resourceVersion: "20271" uid: c6070538-f7a0-4918-b44e-0c9d9cab1730 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58dbb994fc-hbcnb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:26:05Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-fe410-predictor-66c5568bb-9sm5z namespace: kserve-ci-e2e-test resourceVersion: "20285" uid: 1d11f25d-fee7-4da2-84fa-2de26a1f4cb5 kind: Event lastTimestamp: "2026-04-20T20:26:05Z" message: Add eth0 [10.132.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-20T20:26:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-20T20:26:05Z" name: success-200-isvc-fe410-predictor-66c5568bb-9sm5z.18a82a7c2d354dcf namespace: kserve-ci-e2e-test resourceVersion: "20320" uid: 9996628e-1577-4f65-a0b2-b4b3fa31fcca reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:26:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fe410-predictor-66c5568bb-9sm5z namespace: kserve-ci-e2e-test resourceVersion: "20269" uid: 1d11f25d-fee7-4da2-84fa-2de26a1f4cb5 kind: Event lastTimestamp: "2026-04-20T20:26:05Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-20T20:26:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:26:05Z" name: success-200-isvc-fe410-predictor-66c5568bb-9sm5z.18a82a7c2e6961e9 namespace: kserve-ci-e2e-test resourceVersion: "20323" uid: 63167957-9722-4fbd-91dc-fc5541c6ce58 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:26:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fe410-predictor-66c5568bb-9sm5z namespace: kserve-ci-e2e-test resourceVersion: "20269" uid: 1d11f25d-fee7-4da2-84fa-2de26a1f4cb5 kind: Event lastTimestamp: "2026-04-20T20:26:05Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-20T20:26:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:26:05Z" name: success-200-isvc-fe410-predictor-66c5568bb-9sm5z.18a82a7c310bee5c namespace: kserve-ci-e2e-test resourceVersion: "20325" uid: 765f3572-6261-4f99-a5d7-689177a2f990 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:26:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fe410-predictor-66c5568bb-9sm5z namespace: kserve-ci-e2e-test resourceVersion: "20269" uid: 1d11f25d-fee7-4da2-84fa-2de26a1f4cb5 kind: Event lastTimestamp: "2026-04-20T20:26:05Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-20T20:26:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:26:05Z" name: success-200-isvc-fe410-predictor-66c5568bb-9sm5z.18a82a7c317b2d24 namespace: kserve-ci-e2e-test resourceVersion: "20326" uid: 6110867d-f611-416d-9b4d-ba2a8f18494f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:26:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fe410-predictor-66c5568bb-9sm5z namespace: kserve-ci-e2e-test resourceVersion: "20269" uid: 1d11f25d-fee7-4da2-84fa-2de26a1f4cb5 kind: Event lastTimestamp: "2026-04-20T20:26:05Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-20T20:26:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:26:05Z" name: success-200-isvc-fe410-predictor-66c5568bb-9sm5z.18a82a7c3185a506 namespace: kserve-ci-e2e-test resourceVersion: "20327" uid: de689632-d6ca-4b79-b0c9-e60d757ab4b7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:26:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fe410-predictor-66c5568bb-9sm5z namespace: kserve-ci-e2e-test resourceVersion: "20269" uid: 1d11f25d-fee7-4da2-84fa-2de26a1f4cb5 kind: Event lastTimestamp: "2026-04-20T20:26:05Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-20T20:26:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:26:05Z" name: success-200-isvc-fe410-predictor-66c5568bb-9sm5z.18a82a7c33f409c7 namespace: kserve-ci-e2e-test resourceVersion: "20328" uid: b7c03d21-b68b-4563-9dc0-56a0ccffe27e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:26:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fe410-predictor-66c5568bb-9sm5z namespace: kserve-ci-e2e-test resourceVersion: "20269" uid: 1d11f25d-fee7-4da2-84fa-2de26a1f4cb5 kind: Event lastTimestamp: "2026-04-20T20:26:05Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:26:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:26:05Z" name: success-200-isvc-fe410-predictor-66c5568bb-9sm5z.18a82a7c3465bbf9 namespace: kserve-ci-e2e-test resourceVersion: "20329" uid: a5032d9d-3d3a-4c94-9fb9-dfb7f2f138c9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-20T20:26:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fe410-predictor-66c5568bb-9sm5z namespace: kserve-ci-e2e-test resourceVersion: "20269" uid: 1d11f25d-fee7-4da2-84fa-2de26a1f4cb5 kind: Event lastTimestamp: "2026-04-20T20:26:43Z" message: 'Readiness probe failed: dial tcp 10.132.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-20T20:26:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:26:43Z" name: success-200-isvc-fe410-predictor-66c5568bb-9sm5z.18a82a7c8e98c64c namespace: kserve-ci-e2e-test resourceVersion: "20719" uid: 9ed274af-f128-443e-9887-365d5fe50bd3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fe410-predictor-66c5568bb-9sm5z namespace: kserve-ci-e2e-test resourceVersion: "20269" uid: 1d11f25d-fee7-4da2-84fa-2de26a1f4cb5 kind: Event lastTimestamp: "2026-04-20T20:35:20Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-20T20:35:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:35:20Z" name: success-200-isvc-fe410-predictor-66c5568bb-9sm5z.18a82afd4726b08f namespace: kserve-ci-e2e-test resourceVersion: "24541" uid: b91a224c-c8bc-415a-9b06-74a2911b2392 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:35:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fe410-predictor-66c5568bb-9sm5z namespace: kserve-ci-e2e-test resourceVersion: "20269" uid: 1d11f25d-fee7-4da2-84fa-2de26a1f4cb5 kind: Event lastTimestamp: "2026-04-20T20:35:20Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:35:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:35:20Z" name: success-200-isvc-fe410-predictor-66c5568bb-9sm5z.18a82afd472782ce namespace: kserve-ci-e2e-test resourceVersion: "24543" uid: 3ad06530-a30f-4124-bdd6-d1cb1454f7c2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-19.ec2.internal source: component: kubelet host: ip-10-0-140-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:26:05Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-fe410-predictor-66c5568bb namespace: kserve-ci-e2e-test resourceVersion: "20254" uid: a516e6c7-83c0-4394-9c59-1f036c21bc08 kind: Event lastTimestamp: "2026-04-20T20:26:05Z" message: 'Created pod: success-200-isvc-fe410-predictor-66c5568bb-9sm5z' metadata: creationTimestamp: "2026-04-20T20:26:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:26:05Z" name: success-200-isvc-fe410-predictor-66c5568bb.18a82a7c13e2477f namespace: kserve-ci-e2e-test resourceVersion: "20268" uid: 5ce435e9-89a4-49b0-adca-d2f3796e8473 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:26:05Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-fe410-predictor namespace: kserve-ci-e2e-test resourceVersion: "20247" uid: 153ece7a-d4bc-4d44-95ea-21e4d5587a0e kind: Event lastTimestamp: "2026-04-20T20:26:05Z" message: Scaled up replica set success-200-isvc-fe410-predictor-66c5568bb from 0 to 1 metadata: creationTimestamp: "2026-04-20T20:26:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:26:05Z" name: success-200-isvc-fe410-predictor.18a82a7c11ee2301 namespace: kserve-ci-e2e-test resourceVersion: "20256" uid: 04412385-34c1-4028-8aaf-18ec8789cbda reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:26:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fe410-predictor namespace: kserve-ci-e2e-test resourceVersion: "20253" uid: d79f8ae9-3234-4e08-8029-00693d02f27e kind: Event lastTimestamp: "2026-04-20T20:26:35Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:26:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:26:35Z" name: success-200-isvc-fe410-predictor.18a82a7f90d57e6b namespace: kserve-ci-e2e-test resourceVersion: "20654" uid: 0bc1c423-97f2-449e-afb6-86b6ca2322d3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:26:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fe410-predictor namespace: kserve-ci-e2e-test resourceVersion: "20253" uid: d79f8ae9-3234-4e08-8029-00693d02f27e kind: Event lastTimestamp: "2026-04-20T20:26:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:26:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:26:35Z" name: success-200-isvc-fe410-predictor.18a82a7f9136d20c namespace: kserve-ci-e2e-test resourceVersion: "20655" uid: 2b5d878c-9711-45ba-bbdb-f2ca555371bb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-20T20:26:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fe410-predictor namespace: kserve-ci-e2e-test resourceVersion: "20484" uid: d79f8ae9-3234-4e08-8029-00693d02f27e kind: Event lastTimestamp: "2026-04-20T20:27:20Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:26:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:27:20Z" name: success-200-isvc-fe410-predictor.18a82a868ef0a2ed namespace: kserve-ci-e2e-test resourceVersion: "21045" uid: 92562f4c-1361-456b-abe0-5fe282a901f6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-20T20:26:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fe410-predictor namespace: kserve-ci-e2e-test resourceVersion: "20484" uid: d79f8ae9-3234-4e08-8029-00693d02f27e kind: Event lastTimestamp: "2026-04-20T20:27:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:26:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:27:20Z" name: success-200-isvc-fe410-predictor.18a82a868f585c4c namespace: kserve-ci-e2e-test resourceVersion: "21046" uid: 457307a2-bbb1-408b-bb60-b76474d7aecb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-20T20:26:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-fe410 namespace: kserve-ci-e2e-test resourceVersion: "20233" uid: cb981050-49bb-4186-9f18-eb1f324f085e kind: Event lastTimestamp: "2026-04-20T20:26:53Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-20T20:26:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:26:53Z" name: success-200-isvc-fe410.18a82a7c11f8f7cf namespace: kserve-ci-e2e-test resourceVersion: "20799" uid: 2038db48-6c10-49a7-9a0a-6a05e830efa8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:26:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-fe410 namespace: kserve-ci-e2e-test resourceVersion: "20233" uid: cb981050-49bb-4186-9f18-eb1f324f085e kind: Event lastTimestamp: "2026-04-20T20:26:05Z" message: 'Failed to update status for InferenceService "success-200-isvc-fe410": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-fe410": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-20T20:26:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:26:05Z" name: success-200-isvc-fe410.18a82a7c128aa769 namespace: kserve-ci-e2e-test resourceVersion: "20260" uid: a6f1edba-10d0-4ed4-8bf6-d545cfa5d1e0 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:26:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-fe410 namespace: kserve-ci-e2e-test resourceVersion: "20233" uid: cb981050-49bb-4186-9f18-eb1f324f085e kind: Event lastTimestamp: "2026-04-20T20:26:05Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-fe410": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-20T20:26:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:26:05Z" name: success-200-isvc-fe410.18a82a7c128adf43 namespace: kserve-ci-e2e-test resourceVersion: "20261" uid: 23c05178-073f-499b-8cb2-044788de2697 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:26:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-fe410 namespace: kserve-ci-e2e-test resourceVersion: "20797" uid: cb981050-49bb-4186-9f18-eb1f324f085e kind: Event lastTimestamp: "2026-04-20T20:26:53Z" message: InferenceService [success-200-isvc-fe410] is Ready metadata: creationTimestamp: "2026-04-20T20:26:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:26:53Z" name: success-200-isvc-fe410.18a82a8749239dda namespace: kserve-ci-e2e-test resourceVersion: "20798" uid: b811c3b4-2324-45ca-89d4-28e93a208b85 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-20T20:14:49.971060Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-127f1-5bd7f94589-flj29 namespace: kserve-ci-e2e-test resourceVersion: "14276" uid: d5bf96b6-b305-4d4a-a71d-28594dd96238 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-127f1-5bd7f94589-flj29 to ip-10-0-131-234.ec2.internal metadata: creationTimestamp: "2026-04-20T20:14:49Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-20T20:14:49Z" name: switch-graph-127f1-5bd7f94589-flj29.18a829ded1725671 namespace: kserve-ci-e2e-test resourceVersion: "14281" uid: e8f35468-54cd-4148-8f5b-499f9631494f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58dbb994fc-hbcnb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:14:50Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-127f1-5bd7f94589-flj29 namespace: kserve-ci-e2e-test resourceVersion: "14286" uid: d5bf96b6-b305-4d4a-a71d-28594dd96238 kind: Event lastTimestamp: "2026-04-20T20:14:50Z" message: Add eth0 [10.134.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-20T20:14:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-20T20:14:50Z" name: switch-graph-127f1-5bd7f94589-flj29.18a829deec4fa4f3 namespace: kserve-ci-e2e-test resourceVersion: "14296" uid: dac1b108-be48-4858-8423-fb6e74906b59 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:14:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-127f1} kind: Pod name: switch-graph-127f1-5bd7f94589-flj29 namespace: kserve-ci-e2e-test resourceVersion: "14277" uid: d5bf96b6-b305-4d4a-a71d-28594dd96238 kind: Event lastTimestamp: "2026-04-20T20:14:50Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:0169f8fa51f9c0b83c5c9a121f51af9f3e3cdcc48ff1b00c2fcaa12b38a7914f" metadata: creationTimestamp: "2026-04-20T20:14:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:14:50Z" name: switch-graph-127f1-5bd7f94589-flj29.18a829deedbdc3c8 namespace: kserve-ci-e2e-test resourceVersion: "14298" uid: 6faa24cb-5bd3-4340-82bd-26fec2228649 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:14:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-127f1} kind: Pod name: switch-graph-127f1-5bd7f94589-flj29 namespace: kserve-ci-e2e-test resourceVersion: "14277" uid: d5bf96b6-b305-4d4a-a71d-28594dd96238 kind: Event lastTimestamp: "2026-04-20T20:14:52Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:0169f8fa51f9c0b83c5c9a121f51af9f3e3cdcc48ff1b00c2fcaa12b38a7914f" in 2.003s (2.003s including waiting). Image size: 216233553 bytes.' metadata: creationTimestamp: "2026-04-20T20:14:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:14:52Z" name: switch-graph-127f1-5bd7f94589-flj29.18a829df652e232c namespace: kserve-ci-e2e-test resourceVersion: "14314" uid: 600d7b00-a0d1-4cf8-8f87-985375ea5bd5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:14:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-127f1} kind: Pod name: switch-graph-127f1-5bd7f94589-flj29 namespace: kserve-ci-e2e-test resourceVersion: "14277" uid: d5bf96b6-b305-4d4a-a71d-28594dd96238 kind: Event lastTimestamp: "2026-04-20T20:14:52Z" message: 'Created container: switch-graph-127f1' metadata: creationTimestamp: "2026-04-20T20:14:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:14:52Z" name: switch-graph-127f1-5bd7f94589-flj29.18a829df692bf63b namespace: kserve-ci-e2e-test resourceVersion: "14316" uid: 39760638-3e5d-4e98-ae28-0b908f1a8fce reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:14:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-127f1} kind: Pod name: switch-graph-127f1-5bd7f94589-flj29 namespace: kserve-ci-e2e-test resourceVersion: "14277" uid: d5bf96b6-b305-4d4a-a71d-28594dd96238 kind: Event lastTimestamp: "2026-04-20T20:14:52Z" message: Started container switch-graph-127f1 metadata: creationTimestamp: "2026-04-20T20:14:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:14:52Z" name: switch-graph-127f1-5bd7f94589-flj29.18a829df69a70cd1 namespace: kserve-ci-e2e-test resourceVersion: "14317" uid: b38857d8-6a54-4455-93e8-9e7b54818f0b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-127f1} kind: Pod name: switch-graph-127f1-5bd7f94589-flj29 namespace: kserve-ci-e2e-test resourceVersion: "14277" uid: d5bf96b6-b305-4d4a-a71d-28594dd96238 kind: Event lastTimestamp: "2026-04-20T20:15:00Z" message: Stopping container switch-graph-127f1 metadata: creationTimestamp: "2026-04-20T20:15:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:15:00Z" name: switch-graph-127f1-5bd7f94589-flj29.18a829e13090b1d2 namespace: kserve-ci-e2e-test resourceVersion: "14398" uid: 41236b96-e91e-46c0-b3d7-f315ee5e9336 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-20T20:15:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-127f1} kind: Pod name: switch-graph-127f1-5bd7f94589-flj29 namespace: kserve-ci-e2e-test resourceVersion: "14277" uid: d5bf96b6-b305-4d4a-a71d-28594dd96238 kind: Event lastTimestamp: "2026-04-20T20:15:29Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-20T20:15:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:15:29Z" name: switch-graph-127f1-5bd7f94589-flj29.18a829e2242c7920 namespace: kserve-ci-e2e-test resourceVersion: "14775" uid: 77af1b67-dbab-474e-8221-82cb539f62e9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:14:49Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-127f1-5bd7f94589 namespace: kserve-ci-e2e-test resourceVersion: "14273" uid: 3d300e42-aed1-4bf5-b0ea-326d2684ad50 kind: Event lastTimestamp: "2026-04-20T20:14:49Z" message: 'Created pod: switch-graph-127f1-5bd7f94589-flj29' metadata: creationTimestamp: "2026-04-20T20:14:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:14:49Z" name: switch-graph-127f1-5bd7f94589.18a829ded121d64d namespace: kserve-ci-e2e-test resourceVersion: "14278" uid: f321977d-386e-4681-82d3-cad5ebd4deb2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:14:49Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-127f1 namespace: kserve-ci-e2e-test resourceVersion: "14271" uid: d2f400de-ebc0-4ebe-982b-e2aab1d76c20 kind: Event lastTimestamp: "2026-04-20T20:14:49Z" message: Scaled up replica set switch-graph-127f1-5bd7f94589 from 0 to 1 metadata: creationTimestamp: "2026-04-20T20:14:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:14:49Z" name: switch-graph-127f1.18a829ded04684e1 namespace: kserve-ci-e2e-test resourceVersion: "14274" uid: ab2494db-f554-4bdf-a996-d352e8ba8c2d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:14:50Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-127f1 namespace: kserve-ci-e2e-test resourceVersion: "14287" uid: 4a709954-8315-4ba0-98db-9b35a99e7840 kind: Event lastTimestamp: "2026-04-20T20:14:50Z" message: InferenceGraph [switch-graph-127f1] is Ready metadata: creationTimestamp: "2026-04-20T20:14:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:14:50Z" name: switch-graph-127f1.18a829ded6b5764d namespace: kserve-ci-e2e-test resourceVersion: "14288" uid: 342b9f3a-abdf-4cad-be2d-1b6d4ed50ade reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-20T20:36:20.397762Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-60a5c-64b647654d-2knr4 namespace: kserve-ci-e2e-test resourceVersion: "25347" uid: 2fe3de8b-9f73-4927-8756-d868d7db6842 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-60a5c-64b647654d-2knr4 to ip-10-0-131-234.ec2.internal metadata: creationTimestamp: "2026-04-20T20:36:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-20T20:36:20Z" name: switch-graph-60a5c-64b647654d-2knr4.18a82b0b44df2146 namespace: kserve-ci-e2e-test resourceVersion: "25352" uid: 5e8ddb54-c1da-4d5f-ae25-4e84754200bc reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58dbb994fc-hbcnb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:36:20Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-60a5c-64b647654d-2knr4 namespace: kserve-ci-e2e-test resourceVersion: "25356" uid: 2fe3de8b-9f73-4927-8756-d868d7db6842 kind: Event lastTimestamp: "2026-04-20T20:36:20Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-20T20:36:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-20T20:36:20Z" name: switch-graph-60a5c-64b647654d-2knr4.18a82b0b5f04575a namespace: kserve-ci-e2e-test resourceVersion: "25373" uid: 303f15d7-ccbd-4338-a7cf-0e013b1c6406 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:36:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-60a5c} kind: Pod name: switch-graph-60a5c-64b647654d-2knr4 namespace: kserve-ci-e2e-test resourceVersion: "25349" uid: 2fe3de8b-9f73-4927-8756-d868d7db6842 kind: Event lastTimestamp: "2026-04-20T20:36:20Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:0169f8fa51f9c0b83c5c9a121f51af9f3e3cdcc48ff1b00c2fcaa12b38a7914f" already present on machine metadata: creationTimestamp: "2026-04-20T20:36:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:36:20Z" name: switch-graph-60a5c-64b647654d-2knr4.18a82b0b603258f2 namespace: kserve-ci-e2e-test resourceVersion: "25376" uid: ef3e07d5-f306-4f9a-9e60-c010a43bea05 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:36:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-60a5c} kind: Pod name: switch-graph-60a5c-64b647654d-2knr4 namespace: kserve-ci-e2e-test resourceVersion: "25349" uid: 2fe3de8b-9f73-4927-8756-d868d7db6842 kind: Event lastTimestamp: "2026-04-20T20:36:20Z" message: 'Created container: switch-graph-60a5c' metadata: creationTimestamp: "2026-04-20T20:36:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:36:20Z" name: switch-graph-60a5c-64b647654d-2knr4.18a82b0b6352f540 namespace: kserve-ci-e2e-test resourceVersion: "25379" uid: 17edf2d7-39af-47b4-91a3-a5ca0c7786bc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:36:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-60a5c} kind: Pod name: switch-graph-60a5c-64b647654d-2knr4 namespace: kserve-ci-e2e-test resourceVersion: "25349" uid: 2fe3de8b-9f73-4927-8756-d868d7db6842 kind: Event lastTimestamp: "2026-04-20T20:36:20Z" message: Started container switch-graph-60a5c metadata: creationTimestamp: "2026-04-20T20:36:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:36:20Z" name: switch-graph-60a5c-64b647654d-2knr4.18a82b0b63cd834e namespace: kserve-ci-e2e-test resourceVersion: "25381" uid: 31e4a20e-fddf-4b97-b6fc-6e2be070b7fc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:52:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-60a5c} kind: Pod name: switch-graph-60a5c-64b647654d-2knr4 namespace: kserve-ci-e2e-test resourceVersion: "25349" uid: 2fe3de8b-9f73-4927-8756-d868d7db6842 kind: Event lastTimestamp: "2026-04-20T20:52:39Z" message: Stopping container switch-graph-60a5c metadata: creationTimestamp: "2026-04-20T20:52:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:52:39Z" name: switch-graph-60a5c-64b647654d-2knr4.18a82bef52e39b2d namespace: kserve-ci-e2e-test resourceVersion: "32567" uid: ea1d3b03-4ee4-44be-b529-d7066fd0d9d3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-20T20:52:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-60a5c} kind: Pod name: switch-graph-60a5c-64b647654d-2knr4 namespace: kserve-ci-e2e-test resourceVersion: "25349" uid: 2fe3de8b-9f73-4927-8756-d868d7db6842 kind: Event lastTimestamp: "2026-04-20T20:53:02Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-20T20:52:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:53:02Z" name: switch-graph-60a5c-64b647654d-2knr4.18a82befd5aa4a00 namespace: kserve-ci-e2e-test resourceVersion: "32802" uid: 78da3820-f9e3-4c3c-a665-20ca773b7542 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:36:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-60a5c-64b647654d namespace: kserve-ci-e2e-test resourceVersion: "25344" uid: 9437b007-f51e-4bed-a468-b0d492bbcdcf kind: Event lastTimestamp: "2026-04-20T20:36:20Z" message: 'Created pod: switch-graph-60a5c-64b647654d-2knr4' metadata: creationTimestamp: "2026-04-20T20:36:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:36:20Z" name: switch-graph-60a5c-64b647654d.18a82b0b4470f487 namespace: kserve-ci-e2e-test resourceVersion: "25348" uid: 2a42b853-4f85-41c7-941a-4df8ac38bad5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:36:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-60a5c namespace: kserve-ci-e2e-test resourceVersion: "25342" uid: c59fa526-6407-46d6-b994-ee182a395360 kind: Event lastTimestamp: "2026-04-20T20:36:20Z" message: Scaled up replica set switch-graph-60a5c-64b647654d from 0 to 1 metadata: creationTimestamp: "2026-04-20T20:36:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:36:20Z" name: switch-graph-60a5c.18a82b0b43e40456 namespace: kserve-ci-e2e-test resourceVersion: "25345" uid: 45465a0f-9a17-4e35-a35d-3042454b46ac reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:36:20Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-60a5c namespace: kserve-ci-e2e-test resourceVersion: "25358" uid: dfe57aba-00ed-4950-91a4-dcf7f891b71d kind: Event lastTimestamp: "2026-04-20T20:36:20Z" message: InferenceGraph [switch-graph-60a5c] is Ready metadata: creationTimestamp: "2026-04-20T20:36:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:36:20Z" name: switch-graph-60a5c.18a82b0b4719b6b3 namespace: kserve-ci-e2e-test resourceVersion: "25360" uid: d02c7709-4145-4760-a66b-eef0cdc2df95 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:36:20Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-60a5c namespace: kserve-ci-e2e-test resourceVersion: "25332" uid: dfe57aba-00ed-4950-91a4-dcf7f891b71d kind: Event lastTimestamp: "2026-04-20T20:36:20Z" message: 'Failed to update status for InferenceGraph "switch-graph-60a5c": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-60a5c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-20T20:36:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:36:20Z" name: switch-graph-60a5c.18a82b0b49649966 namespace: kserve-ci-e2e-test resourceVersion: "25364" uid: 9f5f77a0-6869-4371-8a03-fa9fff8c1281 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:36:20Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-60a5c namespace: kserve-ci-e2e-test resourceVersion: "25332" uid: dfe57aba-00ed-4950-91a4-dcf7f891b71d kind: Event lastTimestamp: "2026-04-20T20:36:20Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-60a5c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-20T20:36:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:36:20Z" name: switch-graph-60a5c.18a82b0b4964f377 namespace: kserve-ci-e2e-test resourceVersion: "25365" uid: f6675d8b-288d-49c6-9e09-260f049ce969 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:36:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-60a5c namespace: kserve-ci-e2e-test resourceVersion: "25343" uid: fd491dbc-784d-42ff-8931-65e788339b46 kind: Event lastTimestamp: "2026-04-20T20:36:50Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:36:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:36:50Z" name: switch-graph-60a5c.18a82b0ec24c6127 namespace: kserve-ci-e2e-test resourceVersion: "25622" uid: 5b5abf98-cbdb-45fa-b4cb-39e1b0e6c873 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:36:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-60a5c namespace: kserve-ci-e2e-test resourceVersion: "25343" uid: fd491dbc-784d-42ff-8931-65e788339b46 kind: Event lastTimestamp: "2026-04-20T20:36:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:36:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:36:50Z" name: switch-graph-60a5c.18a82b0ec2adb23e namespace: kserve-ci-e2e-test resourceVersion: "25623" uid: 031eef46-582e-4e01-88f4-ca3d0e016e85 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-20T20:16:00.404289Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-9290e-8cc455c84-hqz47 namespace: kserve-ci-e2e-test resourceVersion: "15275" uid: 7e06f30b-363e-4c4a-aa80-a89ac2540654 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-9290e-8cc455c84-hqz47 to ip-10-0-131-234.ec2.internal metadata: creationTimestamp: "2026-04-20T20:16:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-20T20:16:00Z" name: switch-graph-9290e-8cc455c84-hqz47.18a829ef3798171e namespace: kserve-ci-e2e-test resourceVersion: "15280" uid: 18056c12-e11d-46ce-b0ad-4a0323bd68f1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58dbb994fc-hbcnb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:16:00Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-9290e-8cc455c84-hqz47 namespace: kserve-ci-e2e-test resourceVersion: "15277" uid: 7e06f30b-363e-4c4a-aa80-a89ac2540654 kind: Event lastTimestamp: "2026-04-20T20:16:00Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-9290e-serving-cert" not found' metadata: creationTimestamp: "2026-04-20T20:16:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:16:00Z" name: switch-graph-9290e-8cc455c84-hqz47.18a829ef472716c5 namespace: kserve-ci-e2e-test resourceVersion: "15295" uid: ad8d1c3c-9688-4602-969e-71be4f2debe6 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:16:01Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-9290e-8cc455c84-hqz47 namespace: kserve-ci-e2e-test resourceVersion: "15284" uid: 7e06f30b-363e-4c4a-aa80-a89ac2540654 kind: Event lastTimestamp: "2026-04-20T20:16:01Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-20T20:16:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-20T20:16:01Z" name: switch-graph-9290e-8cc455c84-hqz47.18a829ef74b3dc44 namespace: kserve-ci-e2e-test resourceVersion: "15306" uid: 0033fd8f-6f31-4e94-8902-ab7dd02e9cd5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:16:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-9290e} kind: Pod name: switch-graph-9290e-8cc455c84-hqz47 namespace: kserve-ci-e2e-test resourceVersion: "15277" uid: 7e06f30b-363e-4c4a-aa80-a89ac2540654 kind: Event lastTimestamp: "2026-04-20T20:16:01Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:0169f8fa51f9c0b83c5c9a121f51af9f3e3cdcc48ff1b00c2fcaa12b38a7914f" already present on machine metadata: creationTimestamp: "2026-04-20T20:16:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:16:01Z" name: switch-graph-9290e-8cc455c84-hqz47.18a829ef75e5e0a1 namespace: kserve-ci-e2e-test resourceVersion: "15308" uid: 9ef4ad0a-8c89-44f9-b1c3-9113f77375df reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:16:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-9290e} kind: Pod name: switch-graph-9290e-8cc455c84-hqz47 namespace: kserve-ci-e2e-test resourceVersion: "15277" uid: 7e06f30b-363e-4c4a-aa80-a89ac2540654 kind: Event lastTimestamp: "2026-04-20T20:16:01Z" message: 'Created container: switch-graph-9290e' metadata: creationTimestamp: "2026-04-20T20:16:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:16:01Z" name: switch-graph-9290e-8cc455c84-hqz47.18a829ef792f8756 namespace: kserve-ci-e2e-test resourceVersion: "15309" uid: 154bcd2a-aee8-4fbb-8694-b8528c48d988 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:16:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-9290e} kind: Pod name: switch-graph-9290e-8cc455c84-hqz47 namespace: kserve-ci-e2e-test resourceVersion: "15277" uid: 7e06f30b-363e-4c4a-aa80-a89ac2540654 kind: Event lastTimestamp: "2026-04-20T20:16:01Z" message: Started container switch-graph-9290e metadata: creationTimestamp: "2026-04-20T20:16:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:16:01Z" name: switch-graph-9290e-8cc455c84-hqz47.18a829ef79a5a264 namespace: kserve-ci-e2e-test resourceVersion: "15310" uid: 029a9138-0886-46c7-a578-7820c6cffad7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:24:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-9290e} kind: Pod name: switch-graph-9290e-8cc455c84-hqz47 namespace: kserve-ci-e2e-test resourceVersion: "15277" uid: 7e06f30b-363e-4c4a-aa80-a89ac2540654 kind: Event lastTimestamp: "2026-04-20T20:24:15Z" message: Stopping container switch-graph-9290e metadata: creationTimestamp: "2026-04-20T20:24:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:24:15Z" name: switch-graph-9290e-8cc455c84-hqz47.18a82a626c4fefcf namespace: kserve-ci-e2e-test resourceVersion: "18809" uid: 937dc9fe-ddb1-4faf-99b9-f4f43529b2b1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-20T20:24:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-9290e} kind: Pod name: switch-graph-9290e-8cc455c84-hqz47 namespace: kserve-ci-e2e-test resourceVersion: "15277" uid: 7e06f30b-363e-4c4a-aa80-a89ac2540654 kind: Event lastTimestamp: "2026-04-20T20:24:43Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-20T20:24:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:24:43Z" name: switch-graph-9290e-8cc455c84-hqz47.18a82a632e6a8e6e namespace: kserve-ci-e2e-test resourceVersion: "19146" uid: 23272ae7-5e60-4427-bbbb-6865f7f375f7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-234.ec2.internal source: component: kubelet host: ip-10-0-131-234.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:16:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-9290e-8cc455c84 namespace: kserve-ci-e2e-test resourceVersion: "15272" uid: 50f7e252-3a38-4506-aa31-ff832bd366e8 kind: Event lastTimestamp: "2026-04-20T20:16:00Z" message: 'Created pod: switch-graph-9290e-8cc455c84-hqz47' metadata: creationTimestamp: "2026-04-20T20:16:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:16:00Z" name: switch-graph-9290e-8cc455c84.18a829ef372c2043 namespace: kserve-ci-e2e-test resourceVersion: "15276" uid: 50ed7a20-bd96-471e-af08-0a1055cd6769 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:16:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-9290e namespace: kserve-ci-e2e-test resourceVersion: "15270" uid: 97330a02-58ec-464c-b82e-fe087720cf0a kind: Event lastTimestamp: "2026-04-20T20:16:00Z" message: Scaled up replica set switch-graph-9290e-8cc455c84 from 0 to 1 metadata: creationTimestamp: "2026-04-20T20:16:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:16:00Z" name: switch-graph-9290e.18a829ef3672988e namespace: kserve-ci-e2e-test resourceVersion: "15273" uid: d1c0f8b0-cd46-4f35-bd13-d864e0254f86 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:16:00Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-9290e namespace: kserve-ci-e2e-test resourceVersion: "15287" uid: 5e1a662a-a34a-4db9-ab10-44c1bbdae331 kind: Event lastTimestamp: "2026-04-20T20:16:00Z" message: InferenceGraph [switch-graph-9290e] is Ready metadata: creationTimestamp: "2026-04-20T20:16:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:16:00Z" name: switch-graph-9290e.18a829ef3b73c3ec namespace: kserve-ci-e2e-test resourceVersion: "15290" uid: 248185e8-6d9a-4035-a91b-a86d735f58fb reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-20T20:16:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-9290e namespace: kserve-ci-e2e-test resourceVersion: "15271" uid: 6f44d721-e9fa-4837-8a77-ce0161fc578d kind: Event lastTimestamp: "2026-04-20T20:16:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:16:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:16:45Z" name: switch-graph-9290e.18a829f2b4cd59cf namespace: kserve-ci-e2e-test resourceVersion: "15711" uid: 989ab7f4-6368-4edc-bc95-aee170f98d6a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-20T20:16:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-9290e namespace: kserve-ci-e2e-test resourceVersion: "15271" uid: 6f44d721-e9fa-4837-8a77-ce0161fc578d kind: Event lastTimestamp: "2026-04-20T20:16:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:16:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:16:45Z" name: switch-graph-9290e.18a829f2b53751b3 namespace: kserve-ci-e2e-test resourceVersion: "15712" uid: 82da7647-479b-4d59-bf0b-96a3bfd03cb9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning kind: EventList metadata: resourceVersion: "32807"