--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-05-25T13:27:26.260928Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-3e647-56d7ff6db-8dvvb namespace: kserve-ci-e2e-test resourceVersion: "21170" uid: c6c6b443-e7cb-4333-8376-42f13d8e74a0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-3e647-56d7ff6db-8dvvb to ip-10-0-139-18.ec2.internal metadata: creationTimestamp: "2026-05-25T13:27:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T13:27:26Z" name: ensemble-graph-3e647-56d7ff6db-8dvvb.18b2d1f370966e82 namespace: kserve-ci-e2e-test resourceVersion: "21177" uid: 073961ca-d722-43e1-9137-f537ab8aaef4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c46b5f76-4gnkb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:27:26Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-3e647-56d7ff6db-8dvvb namespace: kserve-ci-e2e-test resourceVersion: "21174" uid: c6c6b443-e7cb-4333-8376-42f13d8e74a0 kind: Event lastTimestamp: "2026-05-25T13:27:26Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-3e647-serving-cert" not found' metadata: creationTimestamp: "2026-05-25T13:27:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:27:26Z" name: ensemble-graph-3e647-56d7ff6db-8dvvb.18b2d1f37a508443 namespace: kserve-ci-e2e-test resourceVersion: "21192" uid: 9ff92828-d83d-4a0c-b2bb-ca76417f4f05 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:27:27Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-3e647-56d7ff6db-8dvvb namespace: kserve-ci-e2e-test resourceVersion: "21181" uid: c6c6b443-e7cb-4333-8376-42f13d8e74a0 kind: Event lastTimestamp: "2026-05-25T13:27:27Z" message: Add eth0 [10.132.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T13:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T13:27:27Z" name: ensemble-graph-3e647-56d7ff6db-8dvvb.18b2d1f3ae5275d8 namespace: kserve-ci-e2e-test resourceVersion: "21203" uid: 784881b9-62c4-4f97-b3e4-7a70eb4b065d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:27:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-3e647} kind: Pod name: ensemble-graph-3e647-56d7ff6db-8dvvb namespace: kserve-ci-e2e-test resourceVersion: "21174" uid: c6c6b443-e7cb-4333-8376-42f13d8e74a0 kind: Event lastTimestamp: "2026-05-25T13:27:27Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:367463b1d3262f4416182663b2344c09f7ccedc3764563fd3d0aee32ec772aae" already present on machine metadata: creationTimestamp: "2026-05-25T13:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:27:27Z" name: ensemble-graph-3e647-56d7ff6db-8dvvb.18b2d1f3afa01569 namespace: kserve-ci-e2e-test resourceVersion: "21205" uid: 7cad422f-573a-40ac-8e16-37d6f2be388a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:27:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-3e647} kind: Pod name: ensemble-graph-3e647-56d7ff6db-8dvvb namespace: kserve-ci-e2e-test resourceVersion: "21174" uid: c6c6b443-e7cb-4333-8376-42f13d8e74a0 kind: Event lastTimestamp: "2026-05-25T13:27:27Z" message: 'Created container: ensemble-graph-3e647' metadata: creationTimestamp: "2026-05-25T13:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:27:27Z" name: ensemble-graph-3e647-56d7ff6db-8dvvb.18b2d1f3b2c5328a namespace: kserve-ci-e2e-test resourceVersion: "21206" uid: 164acd7b-1865-479d-bd3d-2e42a27f55d5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:27:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-3e647} kind: Pod name: ensemble-graph-3e647-56d7ff6db-8dvvb namespace: kserve-ci-e2e-test resourceVersion: "21174" uid: c6c6b443-e7cb-4333-8376-42f13d8e74a0 kind: Event lastTimestamp: "2026-05-25T13:27:27Z" message: Started container ensemble-graph-3e647 metadata: creationTimestamp: "2026-05-25T13:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:27:27Z" name: ensemble-graph-3e647-56d7ff6db-8dvvb.18b2d1f3b337e003 namespace: kserve-ci-e2e-test resourceVersion: "21207" uid: e2a0cdf1-c73f-46f5-a797-217bcd18c317 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:35:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-3e647} kind: Pod name: ensemble-graph-3e647-56d7ff6db-8dvvb namespace: kserve-ci-e2e-test resourceVersion: "21174" uid: c6c6b443-e7cb-4333-8376-42f13d8e74a0 kind: Event lastTimestamp: "2026-05-25T13:35:40Z" message: Stopping container ensemble-graph-3e647 metadata: creationTimestamp: "2026-05-25T13:35:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:35:40Z" name: ensemble-graph-3e647-56d7ff6db-8dvvb.18b2d2669ad5f9a8 namespace: kserve-ci-e2e-test resourceVersion: "24675" uid: 007fb2e8-a76f-4dab-84fe-6815268fe505 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T13:35:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-3e647} kind: Pod name: ensemble-graph-3e647-56d7ff6db-8dvvb namespace: kserve-ci-e2e-test resourceVersion: "21174" uid: c6c6b443-e7cb-4333-8376-42f13d8e74a0 kind: Event lastTimestamp: "2026-05-25T13:36:09Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-25T13:35:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:36:09Z" name: ensemble-graph-3e647-56d7ff6db-8dvvb.18b2d26760189471 namespace: kserve-ci-e2e-test resourceVersion: "25016" uid: 3ef1e681-948b-41da-8ff0-dba163c55330 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:27:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-3e647-56d7ff6db namespace: kserve-ci-e2e-test resourceVersion: "21167" uid: 9a737dc0-ce71-45b9-8e9b-935c4bc17c7b kind: Event lastTimestamp: "2026-05-25T13:27:26Z" message: 'Created pod: ensemble-graph-3e647-56d7ff6db-8dvvb' metadata: creationTimestamp: "2026-05-25T13:27:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:27:26Z" name: ensemble-graph-3e647-56d7ff6db.18b2d1f36ff808c2 namespace: kserve-ci-e2e-test resourceVersion: "21173" uid: 39e1cb3a-7932-4eb1-9c2f-c4ba1860be86 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:27:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-3e647 namespace: kserve-ci-e2e-test resourceVersion: "21164" uid: 169d1368-bab9-4d57-a238-93873cd5214b kind: Event lastTimestamp: "2026-05-25T13:27:26Z" message: Scaled up replica set ensemble-graph-3e647-56d7ff6db from 0 to 1 metadata: creationTimestamp: "2026-05-25T13:27:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:27:26Z" name: ensemble-graph-3e647.18b2d1f36ec76236 namespace: kserve-ci-e2e-test resourceVersion: "21168" uid: 6e871e5b-9e01-450f-bed9-f5aec24189b8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:27:26Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-3e647 namespace: kserve-ci-e2e-test resourceVersion: "21187" uid: a479adf3-cd91-46ac-8b8d-af80ca9dce3e kind: Event lastTimestamp: "2026-05-25T13:27:26Z" message: InferenceGraph [ensemble-graph-3e647] is Ready metadata: creationTimestamp: "2026-05-25T13:27:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:27:26Z" name: ensemble-graph-3e647.18b2d1f373f32693 namespace: kserve-ci-e2e-test resourceVersion: "21189" uid: 4a40286e-aec4-4d53-b559-89721cb5983e reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T13:27:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-3e647 namespace: kserve-ci-e2e-test resourceVersion: "21166" uid: 209fe2cb-6238-4f4b-af6f-2305e8cca98a kind: Event lastTimestamp: "2026-05-25T13:27:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T13:27:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:27:56Z" name: ensemble-graph-3e647.18b2d1f6ed1f4a3a namespace: kserve-ci-e2e-test resourceVersion: "21450" uid: b69d8dde-6aa8-4e6e-89d4-981e78e9f8db reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T13:27:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-3e647 namespace: kserve-ci-e2e-test resourceVersion: "21166" uid: 209fe2cb-6238-4f4b-af6f-2305e8cca98a kind: Event lastTimestamp: "2026-05-25T13:27:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T13:27:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:27:56Z" name: ensemble-graph-3e647.18b2d1f6ed821c25 namespace: kserve-ci-e2e-test resourceVersion: "21453" uid: 4acec263-92c0-4266-85eb-7dc5b09d963d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-05-25T13:26:15.938920Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-db114-85585bdbb4-6kg6t namespace: kserve-ci-e2e-test resourceVersion: "20203" uid: 9bb4be1b-7d3a-4c98-bc44-f1453b52244d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-db114-85585bdbb4-6kg6t to ip-10-0-139-18.ec2.internal metadata: creationTimestamp: "2026-05-25T13:26:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T13:26:15Z" name: ensemble-graph-db114-85585bdbb4-6kg6t.18b2d1e31111bd97 namespace: kserve-ci-e2e-test resourceVersion: "20208" uid: 1a664f3f-1689-4591-b00c-3ba27824abe8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c46b5f76-4gnkb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:16Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-db114-85585bdbb4-6kg6t namespace: kserve-ci-e2e-test resourceVersion: "20214" uid: 9bb4be1b-7d3a-4c98-bc44-f1453b52244d kind: Event lastTimestamp: "2026-05-25T13:26:16Z" message: Add eth0 [10.132.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T13:26:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T13:26:16Z" name: ensemble-graph-db114-85585bdbb4-6kg6t.18b2d1e32aed4758 namespace: kserve-ci-e2e-test resourceVersion: "20227" uid: bba87441-32ea-4b17-81cf-2708032003bf reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-db114} kind: Pod name: ensemble-graph-db114-85585bdbb4-6kg6t namespace: kserve-ci-e2e-test resourceVersion: "20206" uid: 9bb4be1b-7d3a-4c98-bc44-f1453b52244d kind: Event lastTimestamp: "2026-05-25T13:26:16Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:367463b1d3262f4416182663b2344c09f7ccedc3764563fd3d0aee32ec772aae" already present on machine metadata: creationTimestamp: "2026-05-25T13:26:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:26:16Z" name: ensemble-graph-db114-85585bdbb4-6kg6t.18b2d1e32c51232f namespace: kserve-ci-e2e-test resourceVersion: "20229" uid: 9b0071f6-2c16-4fa1-aa33-a5e3d0e1b8db reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-db114} kind: Pod name: ensemble-graph-db114-85585bdbb4-6kg6t namespace: kserve-ci-e2e-test resourceVersion: "20206" uid: 9bb4be1b-7d3a-4c98-bc44-f1453b52244d kind: Event lastTimestamp: "2026-05-25T13:26:16Z" message: 'Created container: ensemble-graph-db114' metadata: creationTimestamp: "2026-05-25T13:26:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:26:16Z" name: ensemble-graph-db114-85585bdbb4-6kg6t.18b2d1e32f848542 namespace: kserve-ci-e2e-test resourceVersion: "20232" uid: e13d66b1-fde9-46fe-bce5-5f5d9c0a847c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-db114} kind: Pod name: ensemble-graph-db114-85585bdbb4-6kg6t namespace: kserve-ci-e2e-test resourceVersion: "20206" uid: 9bb4be1b-7d3a-4c98-bc44-f1453b52244d kind: Event lastTimestamp: "2026-05-25T13:26:16Z" message: Started container ensemble-graph-db114 metadata: creationTimestamp: "2026-05-25T13:26:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:26:16Z" name: ensemble-graph-db114-85585bdbb4-6kg6t.18b2d1e32ffe15d5 namespace: kserve-ci-e2e-test resourceVersion: "20233" uid: 205102d0-7a7c-4c0a-8bd3-be71e414605b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-db114} kind: Pod name: ensemble-graph-db114-85585bdbb4-6kg6t namespace: kserve-ci-e2e-test resourceVersion: "20206" uid: 9bb4be1b-7d3a-4c98-bc44-f1453b52244d kind: Event lastTimestamp: "2026-05-25T13:26:26Z" message: Stopping container ensemble-graph-db114 metadata: creationTimestamp: "2026-05-25T13:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:26:26Z" name: ensemble-graph-db114-85585bdbb4-6kg6t.18b2d1e56b0745e9 namespace: kserve-ci-e2e-test resourceVersion: "20323" uid: 620843cb-57aa-4182-b276-d10b3d00796a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T13:26:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-db114} kind: Pod name: ensemble-graph-db114-85585bdbb4-6kg6t namespace: kserve-ci-e2e-test resourceVersion: "20206" uid: 9bb4be1b-7d3a-4c98-bc44-f1453b52244d kind: Event lastTimestamp: "2026-05-25T13:26:52Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-25T13:26:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:26:52Z" name: ensemble-graph-db114-85585bdbb4-6kg6t.18b2d1e5dbb9cb75 namespace: kserve-ci-e2e-test resourceVersion: "20716" uid: 6a251d8c-137c-4e2d-bd1b-6892d23a2013 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-db114-85585bdbb4 namespace: kserve-ci-e2e-test resourceVersion: "20199" uid: 46ad3b0e-ebe0-4314-97f6-3a5c72649d97 kind: Event lastTimestamp: "2026-05-25T13:26:15Z" message: 'Created pod: ensemble-graph-db114-85585bdbb4-6kg6t' metadata: creationTimestamp: "2026-05-25T13:26:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:26:15Z" name: ensemble-graph-db114-85585bdbb4.18b2d1e31092a296 namespace: kserve-ci-e2e-test resourceVersion: "20205" uid: 779db716-d054-4f58-8ad8-40e2bcba7ce6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-db114 namespace: kserve-ci-e2e-test resourceVersion: "20196" uid: 32fbbc9c-3327-41cc-a0a2-ebccac0fce95 kind: Event lastTimestamp: "2026-05-25T13:26:15Z" message: Scaled up replica set ensemble-graph-db114-85585bdbb4 from 0 to 1 metadata: creationTimestamp: "2026-05-25T13:26:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:26:15Z" name: ensemble-graph-db114.18b2d1e30f5b6385 namespace: kserve-ci-e2e-test resourceVersion: "20201" uid: a28cae06-88e7-47fd-a96f-64ede8974694 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:16Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-db114 namespace: kserve-ci-e2e-test resourceVersion: "20216" uid: a1db734c-1650-4929-a002-fab9b0a031cd kind: Event lastTimestamp: "2026-05-25T13:26:16Z" message: InferenceGraph [ensemble-graph-db114] is Ready metadata: creationTimestamp: "2026-05-25T13:26:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:26:16Z" name: ensemble-graph-db114.18b2d1e3150a797a namespace: kserve-ci-e2e-test resourceVersion: "20218" uid: 047d2a5e-db68-42ae-9fd1-a738019b318c reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T13:16:31.129631Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-08c99-predictor-559bfc6b95-vpb62 namespace: kserve-ci-e2e-test resourceVersion: "15612" uid: d968d149-a207-4af3-91bf-db1ae7d8b745 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-08c99-predictor-559bfc6b95-vpb62 to ip-10-0-139-18.ec2.internal metadata: creationTimestamp: "2026-05-25T13:16:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T13:16:31Z" name: error-404-isvc-08c99-predictor-559bfc6b95-vpb62.18b2d15ae7b83d73 namespace: kserve-ci-e2e-test resourceVersion: "15616" uid: ff5c146b-bb42-4ada-90ca-574455534061 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c46b5f76-4gnkb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:31Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-08c99-predictor-559bfc6b95-vpb62 namespace: kserve-ci-e2e-test resourceVersion: "15614" uid: d968d149-a207-4af3-91bf-db1ae7d8b745 kind: Event lastTimestamp: "2026-05-25T13:16:31Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-08c99-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-25T13:16:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:16:31Z" name: error-404-isvc-08c99-predictor-559bfc6b95-vpb62.18b2d15af173041d namespace: kserve-ci-e2e-test resourceVersion: "15623" uid: fcc9bbb1-d206-4f65-b606-148972eef0a5 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:32Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-08c99-predictor-559bfc6b95-vpb62 namespace: kserve-ci-e2e-test resourceVersion: "15620" uid: d968d149-a207-4af3-91bf-db1ae7d8b745 kind: Event lastTimestamp: "2026-05-25T13:16:32Z" message: Add eth0 [10.132.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T13:16:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T13:16:32Z" name: error-404-isvc-08c99-predictor-559bfc6b95-vpb62.18b2d15b25650d43 namespace: kserve-ci-e2e-test resourceVersion: "15650" uid: 1aa5ecd3-149f-4fc7-9f44-59cf6912fe08 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-08c99-predictor-559bfc6b95-vpb62 namespace: kserve-ci-e2e-test resourceVersion: "15614" uid: d968d149-a207-4af3-91bf-db1ae7d8b745 kind: Event lastTimestamp: "2026-05-25T13:16:32Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-25T13:16:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:16:32Z" name: error-404-isvc-08c99-predictor-559bfc6b95-vpb62.18b2d15b26bc9b4c namespace: kserve-ci-e2e-test resourceVersion: "15652" uid: f8d1ab20-e674-4693-b06c-373b5558d822 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-08c99-predictor-559bfc6b95-vpb62 namespace: kserve-ci-e2e-test resourceVersion: "15614" uid: d968d149-a207-4af3-91bf-db1ae7d8b745 kind: Event lastTimestamp: "2026-05-25T13:16:32Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T13:16:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:16:32Z" name: error-404-isvc-08c99-predictor-559bfc6b95-vpb62.18b2d15b29e0f84e namespace: kserve-ci-e2e-test resourceVersion: "15653" uid: 8f46725c-90cf-43b5-9325-4655b9433abf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-08c99-predictor-559bfc6b95-vpb62 namespace: kserve-ci-e2e-test resourceVersion: "15614" uid: d968d149-a207-4af3-91bf-db1ae7d8b745 kind: Event lastTimestamp: "2026-05-25T13:16:32Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T13:16:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:16:32Z" name: error-404-isvc-08c99-predictor-559bfc6b95-vpb62.18b2d15b2a57adfc namespace: kserve-ci-e2e-test resourceVersion: "15654" uid: 4dadc80e-dadf-4fbc-adbc-dea34a628dc5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-08c99-predictor-559bfc6b95-vpb62 namespace: kserve-ci-e2e-test resourceVersion: "15614" uid: d968d149-a207-4af3-91bf-db1ae7d8b745 kind: Event lastTimestamp: "2026-05-25T13:16:32Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-25T13:16:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:16:32Z" name: error-404-isvc-08c99-predictor-559bfc6b95-vpb62.18b2d15b2a64563a namespace: kserve-ci-e2e-test resourceVersion: "15655" uid: c07167cc-249d-4e8c-b187-7a0bec34c740 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-08c99-predictor-559bfc6b95-vpb62 namespace: kserve-ci-e2e-test resourceVersion: "15614" uid: d968d149-a207-4af3-91bf-db1ae7d8b745 kind: Event lastTimestamp: "2026-05-25T13:16:32Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T13:16:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:16:32Z" name: error-404-isvc-08c99-predictor-559bfc6b95-vpb62.18b2d15b2d26c368 namespace: kserve-ci-e2e-test resourceVersion: "15657" uid: 248ad30f-09a5-4657-a8da-c9c698ea2e09 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-08c99-predictor-559bfc6b95-vpb62 namespace: kserve-ci-e2e-test resourceVersion: "15614" uid: d968d149-a207-4af3-91bf-db1ae7d8b745 kind: Event lastTimestamp: "2026-05-25T13:16:32Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T13:16:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:16:32Z" name: error-404-isvc-08c99-predictor-559bfc6b95-vpb62.18b2d15b2d9e81f4 namespace: kserve-ci-e2e-test resourceVersion: "15658" uid: 1c26a7d6-fd64-4e00-aa1c-a9538334282c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T13:16:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-08c99-predictor-559bfc6b95-vpb62 namespace: kserve-ci-e2e-test resourceVersion: "15614" uid: d968d149-a207-4af3-91bf-db1ae7d8b745 kind: Event lastTimestamp: "2026-05-25T13:17:09Z" message: 'Readiness probe failed: dial tcp 10.132.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T13:16:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:17:09Z" name: error-404-isvc-08c99-predictor-559bfc6b95-vpb62.18b2d15b8dfa6c5a namespace: kserve-ci-e2e-test resourceVersion: "16039" uid: b308c03a-ec4d-4a5b-a72f-9a69d1ba4aa6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:25:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-08c99-predictor-559bfc6b95-vpb62 namespace: kserve-ci-e2e-test resourceVersion: "15614" uid: d968d149-a207-4af3-91bf-db1ae7d8b745 kind: Event lastTimestamp: "2026-05-25T13:25:45Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T13:25:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:25:45Z" name: error-404-isvc-08c99-predictor-559bfc6b95-vpb62.18b2d1dc136364bd namespace: kserve-ci-e2e-test resourceVersion: "19858" uid: ff088353-ba54-41ac-97b0-a09eacb003e5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:25:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-08c99-predictor-559bfc6b95-vpb62 namespace: kserve-ci-e2e-test resourceVersion: "15614" uid: d968d149-a207-4af3-91bf-db1ae7d8b745 kind: Event lastTimestamp: "2026-05-25T13:25:45Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T13:25:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:25:45Z" name: error-404-isvc-08c99-predictor-559bfc6b95-vpb62.18b2d1dc1363d45e namespace: kserve-ci-e2e-test resourceVersion: "19862" uid: 71cd14b1-3c9f-4eed-9914-3795db9d3ccb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-08c99-predictor-559bfc6b95 namespace: kserve-ci-e2e-test resourceVersion: "15601" uid: 6620b3ab-ba04-4336-9774-e2cab67fc02d kind: Event lastTimestamp: "2026-05-25T13:16:31Z" message: 'Created pod: error-404-isvc-08c99-predictor-559bfc6b95-vpb62' metadata: creationTimestamp: "2026-05-25T13:16:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:16:31Z" name: error-404-isvc-08c99-predictor-559bfc6b95.18b2d15ae74bfe71 namespace: kserve-ci-e2e-test resourceVersion: "15613" uid: 2f4d5736-d568-4985-8af3-a0dc5a08bd74 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-08c99-predictor namespace: kserve-ci-e2e-test resourceVersion: "15599" uid: 7f29deca-8010-4755-aa05-8ada81783137 kind: Event lastTimestamp: "2026-05-25T13:16:31Z" message: Scaled up replica set error-404-isvc-08c99-predictor-559bfc6b95 from 0 to 1 metadata: creationTimestamp: "2026-05-25T13:16:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:16:31Z" name: error-404-isvc-08c99-predictor.18b2d15ae56b9e35 namespace: kserve-ci-e2e-test resourceVersion: "15604" uid: a7361887-a1e4-481f-b01d-88e432db6c73 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T13:16:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-08c99-predictor namespace: kserve-ci-e2e-test resourceVersion: "15600" uid: 73d29e27-27f6-47d8-8c59-a244230b678a kind: Event lastTimestamp: "2026-05-25T13:17:16Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T13:16:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:17:16Z" name: error-404-isvc-08c99-predictor.18b2d15e63dc4d19 namespace: kserve-ci-e2e-test resourceVersion: "16090" uid: 78f911ad-9311-4802-887e-9fc20bffc843 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T13:16:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-08c99-predictor namespace: kserve-ci-e2e-test resourceVersion: "15600" uid: 73d29e27-27f6-47d8-8c59-a244230b678a kind: Event lastTimestamp: "2026-05-25T13:17:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T13:16:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:17:16Z" name: error-404-isvc-08c99-predictor.18b2d15e64343e9d namespace: kserve-ci-e2e-test resourceVersion: "16092" uid: 5c1647a5-515e-4b44-960b-9a4ab1e6b273 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T13:17:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-08c99-predictor namespace: kserve-ci-e2e-test resourceVersion: "15792" uid: 73d29e27-27f6-47d8-8c59-a244230b678a kind: Event lastTimestamp: "2026-05-25T13:17:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T13:17:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:17:46Z" name: error-404-isvc-08c99-predictor.18b2d168e0b06c68 namespace: kserve-ci-e2e-test resourceVersion: "16368" uid: 731212c9-8748-4678-9309-1b7348092a87 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T13:17:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-08c99-predictor namespace: kserve-ci-e2e-test resourceVersion: "15792" uid: 73d29e27-27f6-47d8-8c59-a244230b678a kind: Event lastTimestamp: "2026-05-25T13:17:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T13:17:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:17:46Z" name: error-404-isvc-08c99-predictor.18b2d168e1319c57 namespace: kserve-ci-e2e-test resourceVersion: "16369" uid: 142d0809-852b-428c-89fa-9aa031640cfb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T13:16:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-08c99 namespace: kserve-ci-e2e-test resourceVersion: "15560" uid: 29642c15-698b-43bd-824d-962e67f37d97 kind: Event lastTimestamp: "2026-05-25T13:17:20Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T13:16:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:17:20Z" name: error-404-isvc-08c99.18b2d15ae561bb21 namespace: kserve-ci-e2e-test resourceVersion: "16134" uid: 29e641ec-2583-4597-8fb3-1270d7517bfe reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-08c99 namespace: kserve-ci-e2e-test resourceVersion: "15560" uid: 29642c15-698b-43bd-824d-962e67f37d97 kind: Event lastTimestamp: "2026-05-25T13:16:31Z" message: 'Failed to update status for InferenceService "error-404-isvc-08c99": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-08c99": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T13:16:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:16:31Z" name: error-404-isvc-08c99.18b2d15ae6186dcc namespace: kserve-ci-e2e-test resourceVersion: "15608" uid: abe7ad8c-fa9e-44a4-8a2e-fe978ac46327 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-08c99 namespace: kserve-ci-e2e-test resourceVersion: "15560" uid: 29642c15-698b-43bd-824d-962e67f37d97 kind: Event lastTimestamp: "2026-05-25T13:16:31Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-08c99": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T13:16:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:16:31Z" name: error-404-isvc-08c99.18b2d15ae618b80a namespace: kserve-ci-e2e-test resourceVersion: "15610" uid: 9c742ade-c323-43e5-8edc-ed8073f41451 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:17:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-08c99 namespace: kserve-ci-e2e-test resourceVersion: "16132" uid: 29642c15-698b-43bd-824d-962e67f37d97 kind: Event lastTimestamp: "2026-05-25T13:17:20Z" message: InferenceService [error-404-isvc-08c99] is Ready metadata: creationTimestamp: "2026-05-25T13:17:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:17:20Z" name: error-404-isvc-08c99.18b2d166495c8d8c namespace: kserve-ci-e2e-test resourceVersion: "16133" uid: 1ab89d09-196d-46c4-a2d8-cb21f50c189b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T13:36:11.211837Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-12313-predictor-7746bc869-vlbsc namespace: kserve-ci-e2e-test resourceVersion: "25131" uid: a7e53380-9d1c-42f5-bbc3-4266185be4a0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-12313-predictor-7746bc869-vlbsc to ip-10-0-139-18.ec2.internal metadata: creationTimestamp: "2026-05-25T13:36:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T13:36:11Z" name: error-404-isvc-12313-predictor-7746bc869-vlbsc.18b2d26daa19b57a namespace: kserve-ci-e2e-test resourceVersion: "25135" uid: 7a2ede80-b9d3-4457-a515-d2b08dd61851 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c46b5f76-4gnkb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:11Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-12313-predictor-7746bc869-vlbsc namespace: kserve-ci-e2e-test resourceVersion: "25133" uid: a7e53380-9d1c-42f5-bbc3-4266185be4a0 kind: Event lastTimestamp: "2026-05-25T13:36:11Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-12313-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-25T13:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:36:11Z" name: error-404-isvc-12313-predictor-7746bc869-vlbsc.18b2d26db544bf93 namespace: kserve-ci-e2e-test resourceVersion: "25143" uid: ef63dd06-f84d-4d3b-886a-de1de4c7f429 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:12Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-12313-predictor-7746bc869-vlbsc namespace: kserve-ci-e2e-test resourceVersion: "25141" uid: a7e53380-9d1c-42f5-bbc3-4266185be4a0 kind: Event lastTimestamp: "2026-05-25T13:36:12Z" message: Add eth0 [10.132.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T13:36:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T13:36:12Z" name: error-404-isvc-12313-predictor-7746bc869-vlbsc.18b2d26de7b668b8 namespace: kserve-ci-e2e-test resourceVersion: "25168" uid: a58c1362-c43a-4568-b7a4-5f9ff4952a28 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-12313-predictor-7746bc869-vlbsc namespace: kserve-ci-e2e-test resourceVersion: "25133" uid: a7e53380-9d1c-42f5-bbc3-4266185be4a0 kind: Event lastTimestamp: "2026-05-25T13:36:12Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-25T13:36:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:36:12Z" name: error-404-isvc-12313-predictor-7746bc869-vlbsc.18b2d26de9627393 namespace: kserve-ci-e2e-test resourceVersion: "25172" uid: 36eef3fd-0e5d-4519-baef-2ffcabab7cfa reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-12313-predictor-7746bc869-vlbsc namespace: kserve-ci-e2e-test resourceVersion: "25133" uid: a7e53380-9d1c-42f5-bbc3-4266185be4a0 kind: Event lastTimestamp: "2026-05-25T13:36:12Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T13:36:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:36:12Z" name: error-404-isvc-12313-predictor-7746bc869-vlbsc.18b2d26decc05500 namespace: kserve-ci-e2e-test resourceVersion: "25173" uid: e5c76b5e-9025-4f3b-ab3e-045c42983022 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-12313-predictor-7746bc869-vlbsc namespace: kserve-ci-e2e-test resourceVersion: "25133" uid: a7e53380-9d1c-42f5-bbc3-4266185be4a0 kind: Event lastTimestamp: "2026-05-25T13:36:12Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T13:36:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:36:12Z" name: error-404-isvc-12313-predictor-7746bc869-vlbsc.18b2d26ded4a13a7 namespace: kserve-ci-e2e-test resourceVersion: "25174" uid: aba2e3b6-734f-4670-8584-eb21296d3ce0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-12313-predictor-7746bc869-vlbsc namespace: kserve-ci-e2e-test resourceVersion: "25133" uid: a7e53380-9d1c-42f5-bbc3-4266185be4a0 kind: Event lastTimestamp: "2026-05-25T13:36:12Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-25T13:36:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:36:12Z" name: error-404-isvc-12313-predictor-7746bc869-vlbsc.18b2d26ded583a04 namespace: kserve-ci-e2e-test resourceVersion: "25175" uid: a582aec7-0ce9-4757-bb2e-40e961b6980a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-12313-predictor-7746bc869-vlbsc namespace: kserve-ci-e2e-test resourceVersion: "25133" uid: a7e53380-9d1c-42f5-bbc3-4266185be4a0 kind: Event lastTimestamp: "2026-05-25T13:36:12Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T13:36:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:36:12Z" name: error-404-isvc-12313-predictor-7746bc869-vlbsc.18b2d26df0174d21 namespace: kserve-ci-e2e-test resourceVersion: "25176" uid: 2d60d629-5b33-4785-82e9-bd3aefcaf5a9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-12313-predictor-7746bc869-vlbsc namespace: kserve-ci-e2e-test resourceVersion: "25133" uid: a7e53380-9d1c-42f5-bbc3-4266185be4a0 kind: Event lastTimestamp: "2026-05-25T13:36:12Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T13:36:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:36:12Z" name: error-404-isvc-12313-predictor-7746bc869-vlbsc.18b2d26df0919710 namespace: kserve-ci-e2e-test resourceVersion: "25177" uid: 22f72b31-542e-4c04-9e3a-ea8f57fb5c1a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T13:36:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-12313-predictor-7746bc869-vlbsc namespace: kserve-ci-e2e-test resourceVersion: "25133" uid: a7e53380-9d1c-42f5-bbc3-4266185be4a0 kind: Event lastTimestamp: "2026-05-25T13:36:49Z" message: 'Readiness probe failed: dial tcp 10.132.0.43:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T13:36:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:36:49Z" name: error-404-isvc-12313-predictor-7746bc869-vlbsc.18b2d26e15c2dd87 namespace: kserve-ci-e2e-test resourceVersion: "25546" uid: 5ab267bf-b7a2-4d94-921a-85119354f659 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:53:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-12313-predictor-7746bc869-vlbsc namespace: kserve-ci-e2e-test resourceVersion: "25133" uid: a7e53380-9d1c-42f5-bbc3-4266185be4a0 kind: Event lastTimestamp: "2026-05-25T13:53:30Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T13:53:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:53:30Z" name: error-404-isvc-12313-predictor-7746bc869-vlbsc.18b2d35fb3e36b85 namespace: kserve-ci-e2e-test resourceVersion: "33131" uid: 5a469f9c-466a-4abc-8c1d-bdb0de9df05a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:53:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-12313-predictor-7746bc869-vlbsc namespace: kserve-ci-e2e-test resourceVersion: "25133" uid: a7e53380-9d1c-42f5-bbc3-4266185be4a0 kind: Event lastTimestamp: "2026-05-25T13:53:30Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T13:53:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:53:30Z" name: error-404-isvc-12313-predictor-7746bc869-vlbsc.18b2d35fb3e44b96 namespace: kserve-ci-e2e-test resourceVersion: "33132" uid: c9bd0b2e-48e7-46a2-b671-ed3b0f29a2c2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:53:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-12313-predictor-7746bc869-vlbsc namespace: kserve-ci-e2e-test resourceVersion: "25133" uid: a7e53380-9d1c-42f5-bbc3-4266185be4a0 kind: Event lastTimestamp: "2026-05-25T13:53:34Z" message: 'Readiness probe failed: Get "https://10.132.0.43:8643/healthz": dial tcp 10.132.0.43:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-25T13:53:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:53:34Z" name: error-404-isvc-12313-predictor-7746bc869-vlbsc.18b2d360766068e9 namespace: kserve-ci-e2e-test resourceVersion: "33191" uid: 9cfc1f6e-1ca2-48a7-b9dc-5a775a14817b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-12313-predictor-7746bc869 namespace: kserve-ci-e2e-test resourceVersion: "25121" uid: 736b6dc8-3a71-41a5-be29-adc5a41bae1d kind: Event lastTimestamp: "2026-05-25T13:36:11Z" message: 'Created pod: error-404-isvc-12313-predictor-7746bc869-vlbsc' metadata: creationTimestamp: "2026-05-25T13:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:36:11Z" name: error-404-isvc-12313-predictor-7746bc869.18b2d26da9aa3fc6 namespace: kserve-ci-e2e-test resourceVersion: "25132" uid: 438570d5-150c-4066-8c9f-cfc061464fa4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-12313-predictor namespace: kserve-ci-e2e-test resourceVersion: "25116" uid: e6cb7729-7d4d-4e8e-945e-2803b0cea7a5 kind: Event lastTimestamp: "2026-05-25T13:36:11Z" message: Scaled up replica set error-404-isvc-12313-predictor-7746bc869 from 0 to 1 metadata: creationTimestamp: "2026-05-25T13:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:36:11Z" name: error-404-isvc-12313-predictor.18b2d26da792ac82 namespace: kserve-ci-e2e-test resourceVersion: "25125" uid: eb8f524b-a4b7-4ef0-b12b-0d683f716008 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-12313-predictor namespace: kserve-ci-e2e-test resourceVersion: "25120" uid: 889a08ee-3517-419d-88dd-7715555b98be kind: Event lastTimestamp: "2026-05-25T13:36:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T13:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:36:26Z" name: error-404-isvc-12313-predictor.18b2d271265f4a45 namespace: kserve-ci-e2e-test resourceVersion: "25309" uid: 73cda8e7-324e-4c90-80f0-fdfc89811572 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-12313-predictor namespace: kserve-ci-e2e-test resourceVersion: "25120" uid: 889a08ee-3517-419d-88dd-7715555b98be kind: Event lastTimestamp: "2026-05-25T13:36:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T13:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:36:26Z" name: error-404-isvc-12313-predictor.18b2d27126b6c8ce namespace: kserve-ci-e2e-test resourceVersion: "25311" uid: bf5dfff9-ea0c-44b7-be26-3517308a59d3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T13:36:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-12313-predictor namespace: kserve-ci-e2e-test resourceVersion: "25308" uid: 889a08ee-3517-419d-88dd-7715555b98be kind: Event lastTimestamp: "2026-05-25T13:37:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T13:36:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:37:11Z" name: error-404-isvc-12313-predictor.18b2d274a6d4b154 namespace: kserve-ci-e2e-test resourceVersion: "26201" uid: 7bf71824-dd4c-43bb-ba9c-daa7065f5aae reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T13:36:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-12313-predictor namespace: kserve-ci-e2e-test resourceVersion: "25308" uid: 889a08ee-3517-419d-88dd-7715555b98be kind: Event lastTimestamp: "2026-05-25T13:37:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T13:36:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:37:11Z" name: error-404-isvc-12313-predictor.18b2d274a7afb164 namespace: kserve-ci-e2e-test resourceVersion: "26202" uid: b9745203-34ef-4a6d-889b-c643ad9fb05f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T13:36:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-12313 namespace: kserve-ci-e2e-test resourceVersion: "25086" uid: dac182c3-a57a-4219-a721-ca9f8de9eaf9 kind: Event lastTimestamp: "2026-05-25T13:36:59Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T13:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:36:59Z" name: error-404-isvc-12313.18b2d26da78ea223 namespace: kserve-ci-e2e-test resourceVersion: "25869" uid: faf3e685-8a4a-499d-9bae-d378fcfa8c69 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-12313 namespace: kserve-ci-e2e-test resourceVersion: "25086" uid: dac182c3-a57a-4219-a721-ca9f8de9eaf9 kind: Event lastTimestamp: "2026-05-25T13:36:11Z" message: 'Failed to update status for InferenceService "error-404-isvc-12313": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-12313": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T13:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:36:11Z" name: error-404-isvc-12313.18b2d26da812b293 namespace: kserve-ci-e2e-test resourceVersion: "25128" uid: 549f7505-c67c-4b75-9459-8785c317243d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-12313 namespace: kserve-ci-e2e-test resourceVersion: "25086" uid: dac182c3-a57a-4219-a721-ca9f8de9eaf9 kind: Event lastTimestamp: "2026-05-25T13:36:11Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-12313": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T13:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:36:11Z" name: error-404-isvc-12313.18b2d26da812ff92 namespace: kserve-ci-e2e-test resourceVersion: "25129" uid: edafd17d-9ff8-488f-b7e8-fac65a74b5b5 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-12313 namespace: kserve-ci-e2e-test resourceVersion: "25866" uid: dac182c3-a57a-4219-a721-ca9f8de9eaf9 kind: Event lastTimestamp: "2026-05-25T13:36:59Z" message: InferenceService [error-404-isvc-12313] is Ready metadata: creationTimestamp: "2026-05-25T13:36:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:36:59Z" name: error-404-isvc-12313.18b2d278d15aeb19 namespace: kserve-ci-e2e-test resourceVersion: "25868" uid: c0eeab36-93a3-42aa-ad93-2dd1eb2e7891 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T13:16:01.104078Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-339a4-predictor-7cdc89ff49-ncgln namespace: kserve-ci-e2e-test resourceVersion: "15141" uid: ff3eadb9-a8c4-43e4-bee2-4e3e1727f2cb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-339a4-predictor-7cdc89ff49-ncgln to ip-10-0-139-18.ec2.internal metadata: creationTimestamp: "2026-05-25T13:16:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T13:16:01Z" name: error-404-isvc-339a4-predictor-7cdc89ff49-ncgln.18b2d153ea0eaf8c namespace: kserve-ci-e2e-test resourceVersion: "15145" uid: deb53914-ab3c-4eb1-8f6f-4c6f3cad4fcf reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c46b5f76-4gnkb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:01Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-339a4-predictor-7cdc89ff49-ncgln namespace: kserve-ci-e2e-test resourceVersion: "15143" uid: ff3eadb9-a8c4-43e4-bee2-4e3e1727f2cb kind: Event lastTimestamp: "2026-05-25T13:16:01Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-339a4-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-25T13:16:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:16:01Z" name: error-404-isvc-339a4-predictor-7cdc89ff49-ncgln.18b2d153f7a1ec1f namespace: kserve-ci-e2e-test resourceVersion: "15158" uid: 6975062f-8104-47a2-b092-5517c91e05c0 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:02Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-339a4-predictor-7cdc89ff49-ncgln namespace: kserve-ci-e2e-test resourceVersion: "15152" uid: ff3eadb9-a8c4-43e4-bee2-4e3e1727f2cb kind: Event lastTimestamp: "2026-05-25T13:16:02Z" message: Add eth0 [10.132.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T13:16:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T13:16:02Z" name: error-404-isvc-339a4-predictor-7cdc89ff49-ncgln.18b2d154278e6bc0 namespace: kserve-ci-e2e-test resourceVersion: "15181" uid: cc9f8a5a-b027-4114-b4e8-5caa8caff184 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-339a4-predictor-7cdc89ff49-ncgln namespace: kserve-ci-e2e-test resourceVersion: "15143" uid: ff3eadb9-a8c4-43e4-bee2-4e3e1727f2cb kind: Event lastTimestamp: "2026-05-25T13:16:02Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-25T13:16:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:16:02Z" name: error-404-isvc-339a4-predictor-7cdc89ff49-ncgln.18b2d154293be05e namespace: kserve-ci-e2e-test resourceVersion: "15183" uid: 36a28518-13ce-46cf-959e-118d04274690 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-339a4-predictor-7cdc89ff49-ncgln namespace: kserve-ci-e2e-test resourceVersion: "15143" uid: ff3eadb9-a8c4-43e4-bee2-4e3e1727f2cb kind: Event lastTimestamp: "2026-05-25T13:16:02Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T13:16:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:16:02Z" name: error-404-isvc-339a4-predictor-7cdc89ff49-ncgln.18b2d1542caee0f0 namespace: kserve-ci-e2e-test resourceVersion: "15185" uid: 2b880d5f-942b-445b-8d6d-1c637f1d414e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-339a4-predictor-7cdc89ff49-ncgln namespace: kserve-ci-e2e-test resourceVersion: "15143" uid: ff3eadb9-a8c4-43e4-bee2-4e3e1727f2cb kind: Event lastTimestamp: "2026-05-25T13:16:02Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T13:16:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:16:02Z" name: error-404-isvc-339a4-predictor-7cdc89ff49-ncgln.18b2d1542d5f0162 namespace: kserve-ci-e2e-test resourceVersion: "15186" uid: 312221e3-707c-4f90-ba72-2dad4b8cb031 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-339a4-predictor-7cdc89ff49-ncgln namespace: kserve-ci-e2e-test resourceVersion: "15143" uid: ff3eadb9-a8c4-43e4-bee2-4e3e1727f2cb kind: Event lastTimestamp: "2026-05-25T13:16:02Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-25T13:16:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:16:02Z" name: error-404-isvc-339a4-predictor-7cdc89ff49-ncgln.18b2d1542d6c2c75 namespace: kserve-ci-e2e-test resourceVersion: "15187" uid: c8752127-4c64-44ad-9af8-4a7d0362edf0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-339a4-predictor-7cdc89ff49-ncgln namespace: kserve-ci-e2e-test resourceVersion: "15143" uid: ff3eadb9-a8c4-43e4-bee2-4e3e1727f2cb kind: Event lastTimestamp: "2026-05-25T13:16:02Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T13:16:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:16:02Z" name: error-404-isvc-339a4-predictor-7cdc89ff49-ncgln.18b2d15430ed3c27 namespace: kserve-ci-e2e-test resourceVersion: "15191" uid: dd444550-e5c1-4895-a2e9-67281363183b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-339a4-predictor-7cdc89ff49-ncgln namespace: kserve-ci-e2e-test resourceVersion: "15143" uid: ff3eadb9-a8c4-43e4-bee2-4e3e1727f2cb kind: Event lastTimestamp: "2026-05-25T13:16:02Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T13:16:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:16:02Z" name: error-404-isvc-339a4-predictor-7cdc89ff49-ncgln.18b2d154316fe3f9 namespace: kserve-ci-e2e-test resourceVersion: "15192" uid: 125f46ef-ff6a-4905-a569-ed4952b1be8d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T13:16:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-339a4-predictor-7cdc89ff49-ncgln namespace: kserve-ci-e2e-test resourceVersion: "15143" uid: ff3eadb9-a8c4-43e4-bee2-4e3e1727f2cb kind: Event lastTimestamp: "2026-05-25T13:16:39Z" message: 'Readiness probe failed: dial tcp 10.132.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T13:16:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:16:39Z" name: error-404-isvc-339a4-predictor-7cdc89ff49-ncgln.18b2d1548b91650c namespace: kserve-ci-e2e-test resourceVersion: "15735" uid: 88e6010a-fdad-45e8-ba7c-5af727420709 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:25:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-339a4-predictor-7cdc89ff49-ncgln namespace: kserve-ci-e2e-test resourceVersion: "15143" uid: ff3eadb9-a8c4-43e4-bee2-4e3e1727f2cb kind: Event lastTimestamp: "2026-05-25T13:25:15Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T13:25:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:25:15Z" name: error-404-isvc-339a4-predictor-7cdc89ff49-ncgln.18b2d1d51034bd82 namespace: kserve-ci-e2e-test resourceVersion: "19476" uid: f8217478-34f6-4e2e-ae89-0222e7196ac0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:25:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-339a4-predictor-7cdc89ff49-ncgln namespace: kserve-ci-e2e-test resourceVersion: "15143" uid: ff3eadb9-a8c4-43e4-bee2-4e3e1727f2cb kind: Event lastTimestamp: "2026-05-25T13:25:15Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T13:25:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:25:15Z" name: error-404-isvc-339a4-predictor-7cdc89ff49-ncgln.18b2d1d510353af0 namespace: kserve-ci-e2e-test resourceVersion: "19478" uid: efd644f6-4bcc-4b42-97e3-deb9717cc958 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-339a4-predictor-7cdc89ff49 namespace: kserve-ci-e2e-test resourceVersion: "15133" uid: e5de1517-cd67-40e2-94f2-e99024cbf243 kind: Event lastTimestamp: "2026-05-25T13:16:01Z" message: 'Created pod: error-404-isvc-339a4-predictor-7cdc89ff49-ncgln' metadata: creationTimestamp: "2026-05-25T13:16:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:16:01Z" name: error-404-isvc-339a4-predictor-7cdc89ff49.18b2d153e984bbff namespace: kserve-ci-e2e-test resourceVersion: "15142" uid: 6c799723-4b41-4a03-b0e6-e05cf494b027 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-339a4-predictor namespace: kserve-ci-e2e-test resourceVersion: "15131" uid: 574771dc-c86d-4253-b3d4-31cdcc200afb kind: Event lastTimestamp: "2026-05-25T13:16:01Z" message: Scaled up replica set error-404-isvc-339a4-predictor-7cdc89ff49 from 0 to 1 metadata: creationTimestamp: "2026-05-25T13:16:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:16:01Z" name: error-404-isvc-339a4-predictor.18b2d153e78804ba namespace: kserve-ci-e2e-test resourceVersion: "15136" uid: 0d5ca78e-624a-42d9-9238-3368dcde59a1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T13:16:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-339a4-predictor namespace: kserve-ci-e2e-test resourceVersion: "15132" uid: 2b457205-9a31-491a-a242-29dc7d478f13 kind: Event lastTimestamp: "2026-05-25T13:16:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T13:16:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:16:31Z" name: error-404-isvc-339a4-predictor.18b2d15765f0a4e1 namespace: kserve-ci-e2e-test resourceVersion: "15603" uid: a1c211a6-acb7-47a3-9700-3b55ebf8c870 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T13:16:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-339a4-predictor namespace: kserve-ci-e2e-test resourceVersion: "15132" uid: 2b457205-9a31-491a-a242-29dc7d478f13 kind: Event lastTimestamp: "2026-05-25T13:16:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T13:16:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:16:31Z" name: error-404-isvc-339a4-predictor.18b2d1576652fca8 namespace: kserve-ci-e2e-test resourceVersion: "15609" uid: dd61922f-0aa8-4e17-9e2f-c3e88d70a948 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T13:16:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-339a4-predictor namespace: kserve-ci-e2e-test resourceVersion: "15330" uid: 2b457205-9a31-491a-a242-29dc7d478f13 kind: Event lastTimestamp: "2026-05-25T13:17:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T13:16:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:17:16Z" name: error-404-isvc-339a4-predictor.18b2d15e63ccc3ea namespace: kserve-ci-e2e-test resourceVersion: "16091" uid: 1ce3511b-9d6d-4693-97ea-80bc393f65e5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T13:16:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-339a4-predictor namespace: kserve-ci-e2e-test resourceVersion: "15330" uid: 2b457205-9a31-491a-a242-29dc7d478f13 kind: Event lastTimestamp: "2026-05-25T13:17:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T13:16:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:17:16Z" name: error-404-isvc-339a4-predictor.18b2d15e64330413 namespace: kserve-ci-e2e-test resourceVersion: "16093" uid: 378fef22-995a-4234-aaa8-39a675f2264e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T13:16:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-339a4 namespace: kserve-ci-e2e-test resourceVersion: "15117" uid: 76b2705d-f521-4f15-8665-86ea5441db62 kind: Event lastTimestamp: "2026-05-25T13:16:49Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T13:16:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:16:49Z" name: error-404-isvc-339a4.18b2d153e75b9eb1 namespace: kserve-ci-e2e-test resourceVersion: "15839" uid: 074a4c91-ec25-4f6a-84e8-2f25c7015165 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-339a4 namespace: kserve-ci-e2e-test resourceVersion: "15117" uid: 76b2705d-f521-4f15-8665-86ea5441db62 kind: Event lastTimestamp: "2026-05-25T13:16:01Z" message: 'Failed to update status for InferenceService "error-404-isvc-339a4": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-339a4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T13:16:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:16:01Z" name: error-404-isvc-339a4.18b2d153e7c504c0 namespace: kserve-ci-e2e-test resourceVersion: "15138" uid: 54d8abde-b179-4913-8cf3-5daf01b432ce reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-339a4 namespace: kserve-ci-e2e-test resourceVersion: "15117" uid: 76b2705d-f521-4f15-8665-86ea5441db62 kind: Event lastTimestamp: "2026-05-25T13:16:01Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-339a4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T13:16:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:16:01Z" name: error-404-isvc-339a4.18b2d153e7c553e0 namespace: kserve-ci-e2e-test resourceVersion: "15139" uid: ba6a4a41-43d5-4387-bcc8-a52db8ecd59f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-339a4 namespace: kserve-ci-e2e-test resourceVersion: "15836" uid: 76b2705d-f521-4f15-8665-86ea5441db62 kind: Event lastTimestamp: "2026-05-25T13:16:49Z" message: InferenceService [error-404-isvc-339a4] is Ready metadata: creationTimestamp: "2026-05-25T13:16:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:16:49Z" name: error-404-isvc-339a4.18b2d15f46b6d802 namespace: kserve-ci-e2e-test resourceVersion: "15837" uid: 1d8cee41-1ef7-4ce1-aeb7-368a0b480d2b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T13:35:41.181061Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-35687-predictor-d8849b54c-bcggv namespace: kserve-ci-e2e-test resourceVersion: "24754" uid: b0756412-f126-47cc-b3cf-fffc1e17e5be kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-35687-predictor-d8849b54c-bcggv to ip-10-0-139-18.ec2.internal metadata: creationTimestamp: "2026-05-25T13:35:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T13:35:41Z" name: error-404-isvc-35687-predictor-d8849b54c-bcggv.18b2d266ac20acd8 namespace: kserve-ci-e2e-test resourceVersion: "24758" uid: 676d09dc-7be8-4cbc-90c5-3fe5ad5c8746 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c46b5f76-4gnkb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:35:41Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-35687-predictor-d8849b54c-bcggv namespace: kserve-ci-e2e-test resourceVersion: "24764" uid: b0756412-f126-47cc-b3cf-fffc1e17e5be kind: Event lastTimestamp: "2026-05-25T13:35:41Z" message: Add eth0 [10.132.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T13:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T13:35:41Z" name: error-404-isvc-35687-predictor-d8849b54c-bcggv.18b2d266c62018e5 namespace: kserve-ci-e2e-test resourceVersion: "24778" uid: 13ea43de-3ff4-47a6-acad-799df4da24f2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:35:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-35687-predictor-d8849b54c-bcggv namespace: kserve-ci-e2e-test resourceVersion: "24756" uid: b0756412-f126-47cc-b3cf-fffc1e17e5be kind: Event lastTimestamp: "2026-05-25T13:35:41Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-25T13:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:35:41Z" name: error-404-isvc-35687-predictor-d8849b54c-bcggv.18b2d266c7989e8d namespace: kserve-ci-e2e-test resourceVersion: "24781" uid: 7072d687-438a-43d5-a204-dd09459ac431 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:35:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-35687-predictor-d8849b54c-bcggv namespace: kserve-ci-e2e-test resourceVersion: "24756" uid: b0756412-f126-47cc-b3cf-fffc1e17e5be kind: Event lastTimestamp: "2026-05-25T13:35:41Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T13:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:35:41Z" name: error-404-isvc-35687-predictor-d8849b54c-bcggv.18b2d266ca77e3f2 namespace: kserve-ci-e2e-test resourceVersion: "24784" uid: c8072586-99f4-4d53-bacb-e88c9175844a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:35:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-35687-predictor-d8849b54c-bcggv namespace: kserve-ci-e2e-test resourceVersion: "24756" uid: b0756412-f126-47cc-b3cf-fffc1e17e5be kind: Event lastTimestamp: "2026-05-25T13:35:41Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T13:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:35:41Z" name: error-404-isvc-35687-predictor-d8849b54c-bcggv.18b2d266caf29f92 namespace: kserve-ci-e2e-test resourceVersion: "24786" uid: d501b016-c2b2-44a6-9a4b-9afa79c924c7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:35:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-35687-predictor-d8849b54c-bcggv namespace: kserve-ci-e2e-test resourceVersion: "24756" uid: b0756412-f126-47cc-b3cf-fffc1e17e5be kind: Event lastTimestamp: "2026-05-25T13:35:41Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-25T13:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:35:41Z" name: error-404-isvc-35687-predictor-d8849b54c-bcggv.18b2d266caff4d2e namespace: kserve-ci-e2e-test resourceVersion: "24788" uid: 094447f5-352e-40a3-b354-2a25fd151e1a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:35:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-35687-predictor-d8849b54c-bcggv namespace: kserve-ci-e2e-test resourceVersion: "24756" uid: b0756412-f126-47cc-b3cf-fffc1e17e5be kind: Event lastTimestamp: "2026-05-25T13:35:41Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T13:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:35:41Z" name: error-404-isvc-35687-predictor-d8849b54c-bcggv.18b2d266cea2ff6b namespace: kserve-ci-e2e-test resourceVersion: "24792" uid: 28996520-3f1d-4b2b-8a2d-3bde498b34dc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:35:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-35687-predictor-d8849b54c-bcggv namespace: kserve-ci-e2e-test resourceVersion: "24756" uid: b0756412-f126-47cc-b3cf-fffc1e17e5be kind: Event lastTimestamp: "2026-05-25T13:35:41Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T13:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:35:41Z" name: error-404-isvc-35687-predictor-d8849b54c-bcggv.18b2d266cf1cc4ac namespace: kserve-ci-e2e-test resourceVersion: "24793" uid: 65d32885-3719-4e4d-a84d-86cf47212bb1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T13:35:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-35687-predictor-d8849b54c-bcggv namespace: kserve-ci-e2e-test resourceVersion: "24756" uid: b0756412-f126-47cc-b3cf-fffc1e17e5be kind: Event lastTimestamp: "2026-05-25T13:36:18Z" message: 'Readiness probe failed: dial tcp 10.132.0.42:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T13:35:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:36:18Z" name: error-404-isvc-35687-predictor-d8849b54c-bcggv.18b2d267128b2674 namespace: kserve-ci-e2e-test resourceVersion: "25247" uid: 42ed8859-162c-4d59-8ee3-461971c4fb98 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-35687-predictor-d8849b54c-bcggv namespace: kserve-ci-e2e-test resourceVersion: "24756" uid: b0756412-f126-47cc-b3cf-fffc1e17e5be kind: Event lastTimestamp: "2026-05-25T13:36:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T13:36:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:36:51Z" name: error-404-isvc-35687-predictor-d8849b54c-bcggv.18b2d276fe320e58 namespace: kserve-ci-e2e-test resourceVersion: "25600" uid: 33212b38-f10a-4130-a6e6-db5ab0a45503 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-35687-predictor-d8849b54c-bcggv namespace: kserve-ci-e2e-test resourceVersion: "24756" uid: b0756412-f126-47cc-b3cf-fffc1e17e5be kind: Event lastTimestamp: "2026-05-25T13:36:51Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T13:36:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:36:51Z" name: error-404-isvc-35687-predictor-d8849b54c-bcggv.18b2d276fe329367 namespace: kserve-ci-e2e-test resourceVersion: "25603" uid: e028fae8-fe88-43b3-9789-832d4bbf1b75 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-35687-predictor-d8849b54c-bcggv namespace: kserve-ci-e2e-test resourceVersion: "24756" uid: b0756412-f126-47cc-b3cf-fffc1e17e5be kind: Event lastTimestamp: "2026-05-25T13:36:53Z" message: 'Readiness probe failed: Get "https://10.132.0.42:8643/healthz": dial tcp 10.132.0.42:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-25T13:36:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:36:53Z" name: error-404-isvc-35687-predictor-d8849b54c-bcggv.18b2d2779aa7b251 namespace: kserve-ci-e2e-test resourceVersion: "25720" uid: 75edd84d-c329-4fa9-8f40-f15d0deea502 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:35:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-35687-predictor-d8849b54c namespace: kserve-ci-e2e-test resourceVersion: "24743" uid: 24cae3d8-3733-438a-8da1-47b6d970dc8e kind: Event lastTimestamp: "2026-05-25T13:35:41Z" message: 'Created pod: error-404-isvc-35687-predictor-d8849b54c-bcggv' metadata: creationTimestamp: "2026-05-25T13:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:35:41Z" name: error-404-isvc-35687-predictor-d8849b54c.18b2d266abb5e4d5 namespace: kserve-ci-e2e-test resourceVersion: "24755" uid: 05e3420b-040f-4f53-a09f-7ff4710b415e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:35:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-35687-predictor namespace: kserve-ci-e2e-test resourceVersion: "24740" uid: 51c373ca-d076-4998-8382-ed37a5c04cc1 kind: Event lastTimestamp: "2026-05-25T13:35:41Z" message: Scaled up replica set error-404-isvc-35687-predictor-d8849b54c from 0 to 1 metadata: creationTimestamp: "2026-05-25T13:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:35:41Z" name: error-404-isvc-35687-predictor.18b2d266a96e422f namespace: kserve-ci-e2e-test resourceVersion: "24746" uid: 875537cd-215f-45e4-b761-12aff3c88d07 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T13:35:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-35687-predictor namespace: kserve-ci-e2e-test resourceVersion: "24742" uid: 3a003257-6fd0-4b1a-b434-99abc523d12a kind: Event lastTimestamp: "2026-05-25T13:36:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T13:35:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:36:41Z" name: error-404-isvc-35687-predictor.18b2d26a27c9b1e2 namespace: kserve-ci-e2e-test resourceVersion: "25475" uid: 6fae9172-3106-4e95-bce8-9070810e8f74 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T13:35:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-35687-predictor namespace: kserve-ci-e2e-test resourceVersion: "24742" uid: 3a003257-6fd0-4b1a-b434-99abc523d12a kind: Event lastTimestamp: "2026-05-25T13:36:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T13:35:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:36:41Z" name: error-404-isvc-35687-predictor.18b2d26a2835e364 namespace: kserve-ci-e2e-test resourceVersion: "25477" uid: d36937be-5cf9-4862-bf64-3fa7f7e6d9cb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-35687-predictor namespace: kserve-ci-e2e-test resourceVersion: "24918" uid: 3a003257-6fd0-4b1a-b434-99abc523d12a kind: Event lastTimestamp: "2026-05-25T13:36:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T13:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:36:11Z" name: error-404-isvc-35687-predictor.18b2d26da6daef06 namespace: kserve-ci-e2e-test resourceVersion: "25118" uid: 716fb913-1a60-47d5-aee1-f887a46de0e4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-35687-predictor namespace: kserve-ci-e2e-test resourceVersion: "24918" uid: 3a003257-6fd0-4b1a-b434-99abc523d12a kind: Event lastTimestamp: "2026-05-25T13:36:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T13:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:36:11Z" name: error-404-isvc-35687-predictor.18b2d26da74e8b9b namespace: kserve-ci-e2e-test resourceVersion: "25122" uid: d5c07d54-20eb-4442-98d1-eebf07e38249 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T13:35:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-35687 namespace: kserve-ci-e2e-test resourceVersion: "24710" uid: f632a6cf-d90a-4e2f-9f83-4f93594f26c4 kind: Event lastTimestamp: "2026-05-25T13:36:29Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T13:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:36:29Z" name: error-404-isvc-35687.18b2d266a9186574 namespace: kserve-ci-e2e-test resourceVersion: "25350" uid: 29cbd604-f9cb-41a8-84bd-13eb3ec11cd8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:35:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-35687 namespace: kserve-ci-e2e-test resourceVersion: "24710" uid: f632a6cf-d90a-4e2f-9f83-4f93594f26c4 kind: Event lastTimestamp: "2026-05-25T13:35:41Z" message: 'Failed to update status for InferenceService "error-404-isvc-35687": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-35687": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T13:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:35:41Z" name: error-404-isvc-35687.18b2d266a999efd3 namespace: kserve-ci-e2e-test resourceVersion: "24749" uid: f62cfc3a-f050-42cc-9011-3a2f2acd59b7 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:35:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-35687 namespace: kserve-ci-e2e-test resourceVersion: "24710" uid: f632a6cf-d90a-4e2f-9f83-4f93594f26c4 kind: Event lastTimestamp: "2026-05-25T13:35:41Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-35687": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T13:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:35:41Z" name: error-404-isvc-35687.18b2d266a99a3472 namespace: kserve-ci-e2e-test resourceVersion: "24751" uid: f8eef922-2bc3-4c73-ad5c-8e1ea27f69b5 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-35687 namespace: kserve-ci-e2e-test resourceVersion: "25347" uid: f632a6cf-d90a-4e2f-9f83-4f93594f26c4 kind: Event lastTimestamp: "2026-05-25T13:36:28Z" message: InferenceService [error-404-isvc-35687] is Ready metadata: creationTimestamp: "2026-05-25T13:36:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:36:28Z" name: error-404-isvc-35687.18b2d271cd5554af namespace: kserve-ci-e2e-test resourceVersion: "25348" uid: 9a435587-01b1-47f8-9fa9-7e2dc06c5909 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T13:26:26.296421Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3e647-predictor-9899bf78c-4dxp6 namespace: kserve-ci-e2e-test resourceVersion: "20398" uid: 09e86a6b-5c8f-4e71-b465-cbe3c22c9ca3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-3e647-predictor-9899bf78c-4dxp6 to ip-10-0-139-18.ec2.internal metadata: creationTimestamp: "2026-05-25T13:26:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T13:26:26Z" name: error-404-isvc-3e647-predictor-9899bf78c-4dxp6.18b2d1e57a6cc3e2 namespace: kserve-ci-e2e-test resourceVersion: "20402" uid: 8051ae6a-86ed-406f-8522-5cd0acef09f3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c46b5f76-4gnkb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:26Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3e647-predictor-9899bf78c-4dxp6 namespace: kserve-ci-e2e-test resourceVersion: "20408" uid: 09e86a6b-5c8f-4e71-b465-cbe3c22c9ca3 kind: Event lastTimestamp: "2026-05-25T13:26:26Z" message: Add eth0 [10.132.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T13:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T13:26:26Z" name: error-404-isvc-3e647-predictor-9899bf78c-4dxp6.18b2d1e5945d8d2d namespace: kserve-ci-e2e-test resourceVersion: "20421" uid: a12da44a-8057-4043-bf63-f25216a0134a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3e647-predictor-9899bf78c-4dxp6 namespace: kserve-ci-e2e-test resourceVersion: "20399" uid: 09e86a6b-5c8f-4e71-b465-cbe3c22c9ca3 kind: Event lastTimestamp: "2026-05-25T13:26:26Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-25T13:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:26:26Z" name: error-404-isvc-3e647-predictor-9899bf78c-4dxp6.18b2d1e5962ac2aa namespace: kserve-ci-e2e-test resourceVersion: "20423" uid: 8d7d2ef8-76b6-4618-9c71-2a0d8cc3ee47 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3e647-predictor-9899bf78c-4dxp6 namespace: kserve-ci-e2e-test resourceVersion: "20399" uid: 09e86a6b-5c8f-4e71-b465-cbe3c22c9ca3 kind: Event lastTimestamp: "2026-05-25T13:26:26Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T13:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:26:26Z" name: error-404-isvc-3e647-predictor-9899bf78c-4dxp6.18b2d1e59916832a namespace: kserve-ci-e2e-test resourceVersion: "20424" uid: ae84e499-d25c-44a6-88ac-3ba6afaa24e1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3e647-predictor-9899bf78c-4dxp6 namespace: kserve-ci-e2e-test resourceVersion: "20399" uid: 09e86a6b-5c8f-4e71-b465-cbe3c22c9ca3 kind: Event lastTimestamp: "2026-05-25T13:26:26Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T13:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:26:26Z" name: error-404-isvc-3e647-predictor-9899bf78c-4dxp6.18b2d1e599928496 namespace: kserve-ci-e2e-test resourceVersion: "20425" uid: 6affc257-8ae8-420a-b527-fe76fa761142 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3e647-predictor-9899bf78c-4dxp6 namespace: kserve-ci-e2e-test resourceVersion: "20399" uid: 09e86a6b-5c8f-4e71-b465-cbe3c22c9ca3 kind: Event lastTimestamp: "2026-05-25T13:26:26Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-25T13:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:26:26Z" name: error-404-isvc-3e647-predictor-9899bf78c-4dxp6.18b2d1e599a12444 namespace: kserve-ci-e2e-test resourceVersion: "20426" uid: 5e29130d-bbd6-4e4e-82ef-e2ca85357861 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3e647-predictor-9899bf78c-4dxp6 namespace: kserve-ci-e2e-test resourceVersion: "20399" uid: 09e86a6b-5c8f-4e71-b465-cbe3c22c9ca3 kind: Event lastTimestamp: "2026-05-25T13:26:26Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T13:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:26:26Z" name: error-404-isvc-3e647-predictor-9899bf78c-4dxp6.18b2d1e59ccf2078 namespace: kserve-ci-e2e-test resourceVersion: "20427" uid: ce3af9b4-a8d5-433b-b93f-df8b36c35b25 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3e647-predictor-9899bf78c-4dxp6 namespace: kserve-ci-e2e-test resourceVersion: "20399" uid: 09e86a6b-5c8f-4e71-b465-cbe3c22c9ca3 kind: Event lastTimestamp: "2026-05-25T13:26:26Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T13:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:26:26Z" name: error-404-isvc-3e647-predictor-9899bf78c-4dxp6.18b2d1e59d44c10b namespace: kserve-ci-e2e-test resourceVersion: "20428" uid: a650adc7-1fb0-4836-9fb9-26b96d9890cc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-05-25T13:26:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3e647-predictor-9899bf78c-4dxp6 namespace: kserve-ci-e2e-test resourceVersion: "20399" uid: 09e86a6b-5c8f-4e71-b465-cbe3c22c9ca3 kind: Event lastTimestamp: "2026-05-25T13:35:42Z" message: 'Readiness probe failed: dial tcp 10.132.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T13:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:35:42Z" name: error-404-isvc-3e647-predictor-9899bf78c-4dxp6.18b2d1e5a1dcb02c namespace: kserve-ci-e2e-test resourceVersion: "24810" uid: ecb8854e-d8a4-46f9-a0f7-11280294f194 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:35:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3e647-predictor-9899bf78c-4dxp6 namespace: kserve-ci-e2e-test resourceVersion: "20399" uid: 09e86a6b-5c8f-4e71-b465-cbe3c22c9ca3 kind: Event lastTimestamp: "2026-05-25T13:35:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T13:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:35:41Z" name: error-404-isvc-3e647-predictor-9899bf78c-4dxp6.18b2d266a737e6f6 namespace: kserve-ci-e2e-test resourceVersion: "24729" uid: 0b10aa32-f5ff-4dae-9f0e-c86aeaf7ebc4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:35:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3e647-predictor-9899bf78c-4dxp6 namespace: kserve-ci-e2e-test resourceVersion: "20399" uid: 09e86a6b-5c8f-4e71-b465-cbe3c22c9ca3 kind: Event lastTimestamp: "2026-05-25T13:35:41Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T13:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:35:41Z" name: error-404-isvc-3e647-predictor-9899bf78c-4dxp6.18b2d266a738904a namespace: kserve-ci-e2e-test resourceVersion: "24734" uid: 0da068be-2dde-46c5-a88b-ec3428bfe5fb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:35:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3e647-predictor-9899bf78c-4dxp6 namespace: kserve-ci-e2e-test resourceVersion: "20399" uid: 09e86a6b-5c8f-4e71-b465-cbe3c22c9ca3 kind: Event lastTimestamp: "2026-05-25T13:35:42Z" message: 'Readiness probe failed: Get "https://10.132.0.37:8643/healthz": dial tcp 10.132.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-25T13:35:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:35:42Z" name: error-404-isvc-3e647-predictor-9899bf78c-4dxp6.18b2d267164ff21e namespace: kserve-ci-e2e-test resourceVersion: "24809" uid: 9c88ffe3-0d36-40f8-b725-3b9897c341d5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-3e647-predictor-9899bf78c namespace: kserve-ci-e2e-test resourceVersion: "20390" uid: 8838188c-bf03-4168-9f0c-867b2541d304 kind: Event lastTimestamp: "2026-05-25T13:26:26Z" message: 'Created pod: error-404-isvc-3e647-predictor-9899bf78c-4dxp6' metadata: creationTimestamp: "2026-05-25T13:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:26:26Z" name: error-404-isvc-3e647-predictor-9899bf78c.18b2d1e579dfb0f6 namespace: kserve-ci-e2e-test resourceVersion: "20400" uid: 23561dfa-2e1c-4068-8cf8-185fe5090775 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-3e647-predictor namespace: kserve-ci-e2e-test resourceVersion: "20387" uid: a19c7574-c465-414d-ba19-9194e7fbaca7 kind: Event lastTimestamp: "2026-05-25T13:26:26Z" message: Scaled up replica set error-404-isvc-3e647-predictor-9899bf78c from 0 to 1 metadata: creationTimestamp: "2026-05-25T13:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:26:26Z" name: error-404-isvc-3e647-predictor.18b2d1e577f8af8d namespace: kserve-ci-e2e-test resourceVersion: "20392" uid: f3e85d81-5aa9-488f-9817-9b77bd80dbff reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-25T13:26:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3e647-predictor namespace: kserve-ci-e2e-test resourceVersion: "20389" uid: d5ce9380-256a-43b1-adf4-366104860589 kind: Event lastTimestamp: "2026-05-25T13:27:56Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T13:26:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:27:56Z" name: error-404-isvc-3e647-predictor.18b2d1e8f659fe6c namespace: kserve-ci-e2e-test resourceVersion: "21476" uid: 96242e5c-64b3-44ea-b4d4-6fefe756633c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-25T13:26:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3e647-predictor namespace: kserve-ci-e2e-test resourceVersion: "20389" uid: d5ce9380-256a-43b1-adf4-366104860589 kind: Event lastTimestamp: "2026-05-25T13:27:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T13:26:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:27:56Z" name: error-404-isvc-3e647-predictor.18b2d1e8f6ba0115 namespace: kserve-ci-e2e-test resourceVersion: "21477" uid: 941c5224-1861-4e65-9f55-289eb81c5cd0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T13:27:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3e647-predictor namespace: kserve-ci-e2e-test resourceVersion: "20591" uid: d5ce9380-256a-43b1-adf4-366104860589 kind: Event lastTimestamp: "2026-05-25T13:27:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T13:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:27:26Z" name: error-404-isvc-3e647-predictor.18b2d1eff423da27 namespace: kserve-ci-e2e-test resourceVersion: "21185" uid: 05f80286-0173-4c03-80b3-3a683476c159 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T13:27:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3e647-predictor namespace: kserve-ci-e2e-test resourceVersion: "20591" uid: d5ce9380-256a-43b1-adf4-366104860589 kind: Event lastTimestamp: "2026-05-25T13:27:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T13:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:27:26Z" name: error-404-isvc-3e647-predictor.18b2d1eff4763d4b namespace: kserve-ci-e2e-test resourceVersion: "21186" uid: ee3d4db6-0e3b-46e0-93fa-c0cbade93b74 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T13:26:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3e647 namespace: kserve-ci-e2e-test resourceVersion: "20357" uid: a95ced21-17e6-4570-a48d-2f8272a9db30 kind: Event lastTimestamp: "2026-05-25T13:27:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T13:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:27:23Z" name: error-404-isvc-3e647.18b2d1e577b0a338 namespace: kserve-ci-e2e-test resourceVersion: "21125" uid: 468fab4d-b28d-4df3-95fb-cd58266a8167 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3e647 namespace: kserve-ci-e2e-test resourceVersion: "20357" uid: a95ced21-17e6-4570-a48d-2f8272a9db30 kind: Event lastTimestamp: "2026-05-25T13:26:26Z" message: 'Failed to update status for InferenceService "error-404-isvc-3e647": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-3e647": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T13:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:26:26Z" name: error-404-isvc-3e647.18b2d1e5781f4688 namespace: kserve-ci-e2e-test resourceVersion: "20394" uid: faba7361-c63f-4bc2-b6ef-b69dae5e696e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3e647 namespace: kserve-ci-e2e-test resourceVersion: "20357" uid: a95ced21-17e6-4570-a48d-2f8272a9db30 kind: Event lastTimestamp: "2026-05-25T13:26:26Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-3e647": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T13:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:26:26Z" name: error-404-isvc-3e647.18b2d1e5781fa02d namespace: kserve-ci-e2e-test resourceVersion: "20396" uid: 804304ef-964f-434e-aa10-e72a79f6a073 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:27:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3e647 namespace: kserve-ci-e2e-test resourceVersion: "21121" uid: a95ced21-17e6-4570-a48d-2f8272a9db30 kind: Event lastTimestamp: "2026-05-25T13:27:23Z" message: InferenceService [error-404-isvc-3e647] is Ready metadata: creationTimestamp: "2026-05-25T13:27:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:27:23Z" name: error-404-isvc-3e647.18b2d1f2b090d815 namespace: kserve-ci-e2e-test resourceVersion: "21122" uid: fbe88ebd-7a87-46c4-a13b-ddbcf7059c36 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T13:26:56.308431Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-72c4b-predictor-549bf86d85-lbvpt namespace: kserve-ci-e2e-test resourceVersion: "20842" uid: f074234f-c787-4e18-ab9c-3cf3e375eecb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-72c4b-predictor-549bf86d85-lbvpt to ip-10-0-139-18.ec2.internal metadata: creationTimestamp: "2026-05-25T13:26:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T13:26:56Z" name: error-404-isvc-72c4b-predictor-549bf86d85-lbvpt.18b2d1ec7747b637 namespace: kserve-ci-e2e-test resourceVersion: "20847" uid: c5266fd1-14ab-4a3b-9ba4-c7d313497ec4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c46b5f76-4gnkb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:56Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-72c4b-predictor-549bf86d85-lbvpt namespace: kserve-ci-e2e-test resourceVersion: "20851" uid: f074234f-c787-4e18-ab9c-3cf3e375eecb kind: Event lastTimestamp: "2026-05-25T13:26:56Z" message: Add eth0 [10.132.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T13:26:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T13:26:56Z" name: error-404-isvc-72c4b-predictor-549bf86d85-lbvpt.18b2d1ec9127a047 namespace: kserve-ci-e2e-test resourceVersion: "20864" uid: 22afbc80-4b47-4bca-921a-6934df6293a0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-72c4b-predictor-549bf86d85-lbvpt namespace: kserve-ci-e2e-test resourceVersion: "20845" uid: f074234f-c787-4e18-ab9c-3cf3e375eecb kind: Event lastTimestamp: "2026-05-25T13:26:56Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-25T13:26:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:26:56Z" name: error-404-isvc-72c4b-predictor-549bf86d85-lbvpt.18b2d1ec92a6c819 namespace: kserve-ci-e2e-test resourceVersion: "20866" uid: 0edccda7-75ef-45f1-a6d7-f38272983077 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-72c4b-predictor-549bf86d85-lbvpt namespace: kserve-ci-e2e-test resourceVersion: "20845" uid: f074234f-c787-4e18-ab9c-3cf3e375eecb kind: Event lastTimestamp: "2026-05-25T13:26:56Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T13:26:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:26:56Z" name: error-404-isvc-72c4b-predictor-549bf86d85-lbvpt.18b2d1ec9586323a namespace: kserve-ci-e2e-test resourceVersion: "20867" uid: b027510c-8a4d-40e5-90d9-47361e7fd638 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-72c4b-predictor-549bf86d85-lbvpt namespace: kserve-ci-e2e-test resourceVersion: "20845" uid: f074234f-c787-4e18-ab9c-3cf3e375eecb kind: Event lastTimestamp: "2026-05-25T13:26:56Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T13:26:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:26:56Z" name: error-404-isvc-72c4b-predictor-549bf86d85-lbvpt.18b2d1ec95f98c20 namespace: kserve-ci-e2e-test resourceVersion: "20868" uid: f722cde5-d143-4c82-b361-a07352497f07 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-72c4b-predictor-549bf86d85-lbvpt namespace: kserve-ci-e2e-test resourceVersion: "20845" uid: f074234f-c787-4e18-ab9c-3cf3e375eecb kind: Event lastTimestamp: "2026-05-25T13:26:56Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-25T13:26:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:26:56Z" name: error-404-isvc-72c4b-predictor-549bf86d85-lbvpt.18b2d1ec9605d48a namespace: kserve-ci-e2e-test resourceVersion: "20869" uid: 470dd03f-1790-4524-9216-6e5a3b40c1e5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-72c4b-predictor-549bf86d85-lbvpt namespace: kserve-ci-e2e-test resourceVersion: "20845" uid: f074234f-c787-4e18-ab9c-3cf3e375eecb kind: Event lastTimestamp: "2026-05-25T13:26:56Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T13:26:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:26:56Z" name: error-404-isvc-72c4b-predictor-549bf86d85-lbvpt.18b2d1ec993c49ba namespace: kserve-ci-e2e-test resourceVersion: "20870" uid: 166c7476-5100-4c8c-beab-0b2c41ea7d5c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-72c4b-predictor-549bf86d85-lbvpt namespace: kserve-ci-e2e-test resourceVersion: "20845" uid: f074234f-c787-4e18-ab9c-3cf3e375eecb kind: Event lastTimestamp: "2026-05-25T13:26:56Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T13:26:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:26:56Z" name: error-404-isvc-72c4b-predictor-549bf86d85-lbvpt.18b2d1ec99b3ae09 namespace: kserve-ci-e2e-test resourceVersion: "20871" uid: f402de6b-0a52-4588-bf44-50bd124a1763 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-05-25T13:26:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-72c4b-predictor-549bf86d85-lbvpt namespace: kserve-ci-e2e-test resourceVersion: "20845" uid: f074234f-c787-4e18-ab9c-3cf3e375eecb kind: Event lastTimestamp: "2026-05-25T13:36:14Z" message: 'Readiness probe failed: dial tcp 10.132.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T13:26:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:36:14Z" name: error-404-isvc-72c4b-predictor-549bf86d85-lbvpt.18b2d1ece15330f3 namespace: kserve-ci-e2e-test resourceVersion: "25203" uid: d2434def-3f84-44a6-8641-d3eb6b8c2778 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-72c4b-predictor-549bf86d85-lbvpt namespace: kserve-ci-e2e-test resourceVersion: "20845" uid: f074234f-c787-4e18-ab9c-3cf3e375eecb kind: Event lastTimestamp: "2026-05-25T13:36:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T13:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:36:11Z" name: error-404-isvc-72c4b-predictor-549bf86d85-lbvpt.18b2d26da7526b0b namespace: kserve-ci-e2e-test resourceVersion: "25123" uid: eb08bca0-4f1e-4923-9b52-a7e7029c09ed reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-72c4b-predictor-549bf86d85-lbvpt namespace: kserve-ci-e2e-test resourceVersion: "20845" uid: f074234f-c787-4e18-ab9c-3cf3e375eecb kind: Event lastTimestamp: "2026-05-25T13:36:11Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T13:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:36:11Z" name: error-404-isvc-72c4b-predictor-549bf86d85-lbvpt.18b2d26da7535902 namespace: kserve-ci-e2e-test resourceVersion: "25127" uid: e986f3d3-9050-4808-abff-a9d73faf5ad1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-72c4b-predictor-549bf86d85-lbvpt namespace: kserve-ci-e2e-test resourceVersion: "20845" uid: f074234f-c787-4e18-ab9c-3cf3e375eecb kind: Event lastTimestamp: "2026-05-25T13:36:14Z" message: 'Readiness probe failed: Get "https://10.132.0.39:8643/healthz": dial tcp 10.132.0.39:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-25T13:36:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:36:14Z" name: error-404-isvc-72c4b-predictor-549bf86d85-lbvpt.18b2d26e559cb964 namespace: kserve-ci-e2e-test resourceVersion: "25202" uid: 0962643b-63a5-4e08-a0e6-43e0f86663e0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-72c4b-predictor-549bf86d85 namespace: kserve-ci-e2e-test resourceVersion: "20834" uid: 82bc84d1-147d-4dc1-b371-9efdee9159c7 kind: Event lastTimestamp: "2026-05-25T13:26:56Z" message: 'Created pod: error-404-isvc-72c4b-predictor-549bf86d85-lbvpt' metadata: creationTimestamp: "2026-05-25T13:26:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:26:56Z" name: error-404-isvc-72c4b-predictor-549bf86d85.18b2d1ec76d83a25 namespace: kserve-ci-e2e-test resourceVersion: "20844" uid: a17e82ab-325d-474c-a398-04671118c5ca reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-72c4b-predictor namespace: kserve-ci-e2e-test resourceVersion: "20831" uid: be210511-e289-4b31-8259-5770b659e484 kind: Event lastTimestamp: "2026-05-25T13:26:56Z" message: Scaled up replica set error-404-isvc-72c4b-predictor-549bf86d85 from 0 to 1 metadata: creationTimestamp: "2026-05-25T13:26:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:26:56Z" name: error-404-isvc-72c4b-predictor.18b2d1ec75471a1c namespace: kserve-ci-e2e-test resourceVersion: "20837" uid: 560061fd-de7c-4ba7-a1c8-2616da1607e4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-25T13:27:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-72c4b-predictor namespace: kserve-ci-e2e-test resourceVersion: "20833" uid: 280f8abe-6818-433a-9629-ef7f3f29dc07 kind: Event lastTimestamp: "2026-05-25T13:27:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T13:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:27:56Z" name: error-404-isvc-72c4b-predictor.18b2d1eff3b5a735 namespace: kserve-ci-e2e-test resourceVersion: "21473" uid: d44be8e1-8ac3-4303-af0d-6bf61d2be22a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-25T13:27:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-72c4b-predictor namespace: kserve-ci-e2e-test resourceVersion: "20833" uid: 280f8abe-6818-433a-9629-ef7f3f29dc07 kind: Event lastTimestamp: "2026-05-25T13:27:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T13:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:27:56Z" name: error-404-isvc-72c4b-predictor.18b2d1eff4131d19 namespace: kserve-ci-e2e-test resourceVersion: "21475" uid: d4347845-e5c4-4c94-b3c3-a93654ecf7a7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:28:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-72c4b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21022" uid: 280f8abe-6818-433a-9629-ef7f3f29dc07 kind: Event lastTimestamp: "2026-05-25T13:28:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T13:28:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:28:11Z" name: error-404-isvc-72c4b-predictor.18b2d1fdef841963 namespace: kserve-ci-e2e-test resourceVersion: "21599" uid: 9b62e6f0-a99a-4fa3-b680-8530f6f2bfd8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:28:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-72c4b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21022" uid: 280f8abe-6818-433a-9629-ef7f3f29dc07 kind: Event lastTimestamp: "2026-05-25T13:28:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T13:28:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:28:11Z" name: error-404-isvc-72c4b-predictor.18b2d1fdefebf200 namespace: kserve-ci-e2e-test resourceVersion: "21601" uid: 1b07a9d1-f031-4ad7-8dba-3a2eda662d88 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T13:26:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-72c4b namespace: kserve-ci-e2e-test resourceVersion: "20815" uid: 3f620dc1-455a-46ef-8b2e-69d9a73ea4d1 kind: Event lastTimestamp: "2026-05-25T13:27:44Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T13:26:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:27:44Z" name: error-404-isvc-72c4b.18b2d1ec7514eb3d namespace: kserve-ci-e2e-test resourceVersion: "21357" uid: 2147b64e-3e07-4bad-8be4-781fb7b09b36 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:27:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-72c4b namespace: kserve-ci-e2e-test resourceVersion: "21355" uid: 3f620dc1-455a-46ef-8b2e-69d9a73ea4d1 kind: Event lastTimestamp: "2026-05-25T13:27:44Z" message: InferenceService [error-404-isvc-72c4b] is Ready metadata: creationTimestamp: "2026-05-25T13:27:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:27:44Z" name: error-404-isvc-72c4b.18b2d1f79c55f70d namespace: kserve-ci-e2e-test resourceVersion: "21356" uid: 9d51c712-a99d-469f-b2de-233a40388c88 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T13:36:51.456355Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-766ab-predictor-76978cfc8f-lfgfl namespace: kserve-ci-e2e-test resourceVersion: "25656" uid: ec4aa36e-fbc2-45e4-ae60-a072f93354c7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-766ab-predictor-76978cfc8f-lfgfl to ip-10-0-139-18.ec2.internal metadata: creationTimestamp: "2026-05-25T13:36:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T13:36:51Z" name: error-404-isvc-766ab-predictor-76978cfc8f-lfgfl.18b2d27708dcbf5e namespace: kserve-ci-e2e-test resourceVersion: "25660" uid: 29d0009f-2b6e-479f-92af-647f0efc195e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c46b5f76-4gnkb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:51Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-766ab-predictor-76978cfc8f-lfgfl namespace: kserve-ci-e2e-test resourceVersion: "25658" uid: ec4aa36e-fbc2-45e4-ae60-a072f93354c7 kind: Event lastTimestamp: "2026-05-25T13:36:51Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-766ab-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-25T13:36:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:36:51Z" name: error-404-isvc-766ab-predictor-76978cfc8f-lfgfl.18b2d27713d4fc93 namespace: kserve-ci-e2e-test resourceVersion: "25673" uid: 5dd404c5-91ea-4f5c-81ed-2648f363105e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:52Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-766ab-predictor-76978cfc8f-lfgfl namespace: kserve-ci-e2e-test resourceVersion: "25666" uid: ec4aa36e-fbc2-45e4-ae60-a072f93354c7 kind: Event lastTimestamp: "2026-05-25T13:36:52Z" message: Add eth0 [10.132.0.45/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T13:36:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T13:36:52Z" name: error-404-isvc-766ab-predictor-76978cfc8f-lfgfl.18b2d277465fdc75 namespace: kserve-ci-e2e-test resourceVersion: "25689" uid: 46633f46-3c73-494e-aec5-11d4ec8f5012 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-766ab-predictor-76978cfc8f-lfgfl namespace: kserve-ci-e2e-test resourceVersion: "25658" uid: ec4aa36e-fbc2-45e4-ae60-a072f93354c7 kind: Event lastTimestamp: "2026-05-25T13:36:52Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-25T13:36:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:36:52Z" name: error-404-isvc-766ab-predictor-76978cfc8f-lfgfl.18b2d27747dca33e namespace: kserve-ci-e2e-test resourceVersion: "25691" uid: 1bf6b77a-7088-40cd-b16d-2790ce22027f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-766ab-predictor-76978cfc8f-lfgfl namespace: kserve-ci-e2e-test resourceVersion: "25658" uid: ec4aa36e-fbc2-45e4-ae60-a072f93354c7 kind: Event lastTimestamp: "2026-05-25T13:36:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T13:36:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:36:52Z" name: error-404-isvc-766ab-predictor-76978cfc8f-lfgfl.18b2d2774b01f12e namespace: kserve-ci-e2e-test resourceVersion: "25696" uid: 1de910d7-6728-44be-9abe-418841dba0a5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-766ab-predictor-76978cfc8f-lfgfl namespace: kserve-ci-e2e-test resourceVersion: "25658" uid: ec4aa36e-fbc2-45e4-ae60-a072f93354c7 kind: Event lastTimestamp: "2026-05-25T13:36:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T13:36:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:36:52Z" name: error-404-isvc-766ab-predictor-76978cfc8f-lfgfl.18b2d2774b7bb2f7 namespace: kserve-ci-e2e-test resourceVersion: "25697" uid: 45f62433-c7e9-4cd6-9403-62e9e31bb5ef reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-766ab-predictor-76978cfc8f-lfgfl namespace: kserve-ci-e2e-test resourceVersion: "25658" uid: ec4aa36e-fbc2-45e4-ae60-a072f93354c7 kind: Event lastTimestamp: "2026-05-25T13:36:52Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-25T13:36:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:36:52Z" name: error-404-isvc-766ab-predictor-76978cfc8f-lfgfl.18b2d2774b8948cd namespace: kserve-ci-e2e-test resourceVersion: "25698" uid: 3dd24881-f9c7-45b4-ab96-658b36e662cb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-766ab-predictor-76978cfc8f-lfgfl namespace: kserve-ci-e2e-test resourceVersion: "25658" uid: ec4aa36e-fbc2-45e4-ae60-a072f93354c7 kind: Event lastTimestamp: "2026-05-25T13:36:52Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T13:36:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:36:52Z" name: error-404-isvc-766ab-predictor-76978cfc8f-lfgfl.18b2d2774eb0210a namespace: kserve-ci-e2e-test resourceVersion: "25700" uid: 30ecee87-4dc3-46ea-8ce5-2a30c6472008 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-766ab-predictor-76978cfc8f-lfgfl namespace: kserve-ci-e2e-test resourceVersion: "25658" uid: ec4aa36e-fbc2-45e4-ae60-a072f93354c7 kind: Event lastTimestamp: "2026-05-25T13:36:52Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T13:36:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:36:52Z" name: error-404-isvc-766ab-predictor-76978cfc8f-lfgfl.18b2d2774f2b5794 namespace: kserve-ci-e2e-test resourceVersion: "25701" uid: d4cbb052-7010-49f7-ba6b-59984d6d6a8d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T13:36:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-766ab-predictor-76978cfc8f-lfgfl namespace: kserve-ci-e2e-test resourceVersion: "25658" uid: ec4aa36e-fbc2-45e4-ae60-a072f93354c7 kind: Event lastTimestamp: "2026-05-25T13:37:30Z" message: 'Readiness probe failed: dial tcp 10.132.0.45:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T13:36:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:37:30Z" name: error-404-isvc-766ab-predictor-76978cfc8f-lfgfl.18b2d277aa459143 namespace: kserve-ci-e2e-test resourceVersion: "26366" uid: 3a95b33d-8e12-4f74-bec7-28f8d03db7e8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:46:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-766ab-predictor-76978cfc8f-lfgfl namespace: kserve-ci-e2e-test resourceVersion: "25658" uid: ec4aa36e-fbc2-45e4-ae60-a072f93354c7 kind: Event lastTimestamp: "2026-05-25T13:46:06Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T13:46:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:46:06Z" name: error-404-isvc-766ab-predictor-76978cfc8f-lfgfl.18b2d2f838e21029 namespace: kserve-ci-e2e-test resourceVersion: "30052" uid: 651eb709-e596-4ad6-8b4e-ff637513647a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:46:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-766ab-predictor-76978cfc8f-lfgfl namespace: kserve-ci-e2e-test resourceVersion: "25658" uid: ec4aa36e-fbc2-45e4-ae60-a072f93354c7 kind: Event lastTimestamp: "2026-05-25T13:46:06Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T13:46:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:46:06Z" name: error-404-isvc-766ab-predictor-76978cfc8f-lfgfl.18b2d2f838e29eb7 namespace: kserve-ci-e2e-test resourceVersion: "30053" uid: ad2c4d5d-20c1-4131-bdb0-40d2cebcd930 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-766ab-predictor-76978cfc8f namespace: kserve-ci-e2e-test resourceVersion: "25649" uid: 959c7b2d-801c-4869-bd9b-1d074435badd kind: Event lastTimestamp: "2026-05-25T13:36:51Z" message: 'Created pod: error-404-isvc-766ab-predictor-76978cfc8f-lfgfl' metadata: creationTimestamp: "2026-05-25T13:36:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:36:51Z" name: error-404-isvc-766ab-predictor-76978cfc8f.18b2d277086cc37b namespace: kserve-ci-e2e-test resourceVersion: "25657" uid: 76901811-a9df-4268-8796-4cc3ed2d46c5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-766ab-predictor namespace: kserve-ci-e2e-test resourceVersion: "25647" uid: f504e2ac-f725-45b5-bc13-b14afaf1c63c kind: Event lastTimestamp: "2026-05-25T13:36:51Z" message: Scaled up replica set error-404-isvc-766ab-predictor-76978cfc8f from 0 to 1 metadata: creationTimestamp: "2026-05-25T13:36:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:36:51Z" name: error-404-isvc-766ab-predictor.18b2d27706cb77f0 namespace: kserve-ci-e2e-test resourceVersion: "25651" uid: 540bb15c-f336-40e7-b42d-bd2bf6195282 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T13:37:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-766ab-predictor namespace: kserve-ci-e2e-test resourceVersion: "25648" uid: 29ff7020-580e-4a4c-bcf9-b5d158b4c1ff kind: Event lastTimestamp: "2026-05-25T13:37:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T13:37:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:37:36Z" name: error-404-isvc-766ab-predictor.18b2d27a854cc668 namespace: kserve-ci-e2e-test resourceVersion: "26413" uid: a591bfe8-fddf-4009-90bd-f9feb042022e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T13:37:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-766ab-predictor namespace: kserve-ci-e2e-test resourceVersion: "25648" uid: 29ff7020-580e-4a4c-bcf9-b5d158b4c1ff kind: Event lastTimestamp: "2026-05-25T13:37:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T13:37:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:37:36Z" name: error-404-isvc-766ab-predictor.18b2d27a85bf1686 namespace: kserve-ci-e2e-test resourceVersion: "26414" uid: ddd90d71-de9d-4bb2-8854-4ec7352d1d5b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T13:37:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-766ab-predictor namespace: kserve-ci-e2e-test resourceVersion: "26111" uid: 29ff7020-580e-4a4c-bcf9-b5d158b4c1ff kind: Event lastTimestamp: "2026-05-25T13:38:06Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T13:37:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:38:06Z" name: error-404-isvc-766ab-predictor.18b2d27e04710842 namespace: kserve-ci-e2e-test resourceVersion: "26693" uid: 0aaa5e62-4617-427b-a0f2-5115b7076f93 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T13:37:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-766ab-predictor namespace: kserve-ci-e2e-test resourceVersion: "26111" uid: 29ff7020-580e-4a4c-bcf9-b5d158b4c1ff kind: Event lastTimestamp: "2026-05-25T13:38:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T13:37:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:38:06Z" name: error-404-isvc-766ab-predictor.18b2d27e04e48374 namespace: kserve-ci-e2e-test resourceVersion: "26694" uid: a0d7c10f-b97c-4b49-bde4-cb19d5222c08 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T13:36:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-766ab namespace: kserve-ci-e2e-test resourceVersion: "25637" uid: 227736f3-95eb-41af-9706-19469f393079 kind: Event lastTimestamp: "2026-05-25T13:37:40Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T13:36:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:37:40Z" name: error-404-isvc-766ab.18b2d277069c215a namespace: kserve-ci-e2e-test resourceVersion: "26455" uid: f2fd4e8c-e9f4-44cc-b8a3-28450cbd11e0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-766ab namespace: kserve-ci-e2e-test resourceVersion: "25637" uid: 227736f3-95eb-41af-9706-19469f393079 kind: Event lastTimestamp: "2026-05-25T13:36:51Z" message: 'Failed to update status for InferenceService "error-404-isvc-766ab": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-766ab": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T13:36:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:36:51Z" name: error-404-isvc-766ab.18b2d27706ee72a6 namespace: kserve-ci-e2e-test resourceVersion: "25653" uid: 89782ab8-cb23-4848-b492-1841dcd1af0e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-766ab namespace: kserve-ci-e2e-test resourceVersion: "25637" uid: 227736f3-95eb-41af-9706-19469f393079 kind: Event lastTimestamp: "2026-05-25T13:36:51Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-766ab": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T13:36:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:36:51Z" name: error-404-isvc-766ab.18b2d27706eebdc8 namespace: kserve-ci-e2e-test resourceVersion: "25654" uid: 0178875e-a4d0-4112-980a-ddba5788149f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:37:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-766ab namespace: kserve-ci-e2e-test resourceVersion: "26452" uid: 227736f3-95eb-41af-9706-19469f393079 kind: Event lastTimestamp: "2026-05-25T13:37:40Z" message: InferenceService [error-404-isvc-766ab] is Ready metadata: creationTimestamp: "2026-05-25T13:37:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:37:40Z" name: error-404-isvc-766ab.18b2d282656195a4 namespace: kserve-ci-e2e-test resourceVersion: "26453" uid: 5d71cb1f-f946-4357-ab30-abd457080110 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T13:14:31.043110Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-889c1-predictor-b649dc755-pwsvf namespace: kserve-ci-e2e-test resourceVersion: "14074" uid: fe91929d-2d86-4a10-9dc9-8ffb2a26434f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-889c1-predictor-b649dc755-pwsvf to ip-10-0-139-18.ec2.internal metadata: creationTimestamp: "2026-05-25T13:14:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T13:14:31Z" name: error-404-isvc-889c1-predictor-b649dc755-pwsvf.18b2d13ef2014c0c namespace: kserve-ci-e2e-test resourceVersion: "14079" uid: 47386743-d2b6-499b-a80d-6079d9e82d3a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c46b5f76-4gnkb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:31Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-889c1-predictor-b649dc755-pwsvf namespace: kserve-ci-e2e-test resourceVersion: "14077" uid: fe91929d-2d86-4a10-9dc9-8ffb2a26434f kind: Event lastTimestamp: "2026-05-25T13:14:31Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-889c1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-25T13:14:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:14:31Z" name: error-404-isvc-889c1-predictor-b649dc755-pwsvf.18b2d13f03b37bb1 namespace: kserve-ci-e2e-test resourceVersion: "14115" uid: d43d7ce1-5d7e-4357-a4e0-28e80a648b3e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:32Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-889c1-predictor-b649dc755-pwsvf namespace: kserve-ci-e2e-test resourceVersion: "14091" uid: fe91929d-2d86-4a10-9dc9-8ffb2a26434f kind: Event lastTimestamp: "2026-05-25T13:14:32Z" message: Add eth0 [10.132.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T13:14:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T13:14:32Z" name: error-404-isvc-889c1-predictor-b649dc755-pwsvf.18b2d13f2fbdd923 namespace: kserve-ci-e2e-test resourceVersion: "14155" uid: 78435cf8-d6ad-4e4e-92c1-eebf16ceb25e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-889c1-predictor-b649dc755-pwsvf namespace: kserve-ci-e2e-test resourceVersion: "14077" uid: fe91929d-2d86-4a10-9dc9-8ffb2a26434f kind: Event lastTimestamp: "2026-05-25T13:14:32Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1532" metadata: creationTimestamp: "2026-05-25T13:14:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:14:32Z" name: error-404-isvc-889c1-predictor-b649dc755-pwsvf.18b2d13f31115529 namespace: kserve-ci-e2e-test resourceVersion: "14158" uid: f0a095f8-b648-4233-8313-1d743af5aeab reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-889c1-predictor-b649dc755-pwsvf namespace: kserve-ci-e2e-test resourceVersion: "14077" uid: fe91929d-2d86-4a10-9dc9-8ffb2a26434f kind: Event lastTimestamp: "2026-05-25T13:14:43Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1532" in 11.127s (11.127s including waiting). Image size: 1334702699 bytes.' metadata: creationTimestamp: "2026-05-25T13:14:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:14:43Z" name: error-404-isvc-889c1-predictor-b649dc755-pwsvf.18b2d141c84dbb65 namespace: kserve-ci-e2e-test resourceVersion: "14294" uid: c3dd29b2-1714-49e3-b1d3-fcfac806f2b1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-889c1-predictor-b649dc755-pwsvf namespace: kserve-ci-e2e-test resourceVersion: "14077" uid: fe91929d-2d86-4a10-9dc9-8ffb2a26434f kind: Event lastTimestamp: "2026-05-25T13:14:43Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T13:14:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:14:43Z" name: error-404-isvc-889c1-predictor-b649dc755-pwsvf.18b2d141cbf4a66b namespace: kserve-ci-e2e-test resourceVersion: "14297" uid: 6f29f787-8836-435a-9bb0-b981f0a04138 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-889c1-predictor-b649dc755-pwsvf namespace: kserve-ci-e2e-test resourceVersion: "14077" uid: fe91929d-2d86-4a10-9dc9-8ffb2a26434f kind: Event lastTimestamp: "2026-05-25T13:14:43Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T13:14:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:14:43Z" name: error-404-isvc-889c1-predictor-b649dc755-pwsvf.18b2d141cc7ab235 namespace: kserve-ci-e2e-test resourceVersion: "14299" uid: d1bbbba3-6748-4a1c-ac97-8bb7d67be8d3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-889c1-predictor-b649dc755-pwsvf namespace: kserve-ci-e2e-test resourceVersion: "14077" uid: fe91929d-2d86-4a10-9dc9-8ffb2a26434f kind: Event lastTimestamp: "2026-05-25T13:14:43Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-05-25T13:14:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:14:43Z" name: error-404-isvc-889c1-predictor-b649dc755-pwsvf.18b2d141cc86b219 namespace: kserve-ci-e2e-test resourceVersion: "14300" uid: bfdb45a3-9790-4dde-b577-54c028d19cfd reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-889c1-predictor-b649dc755-pwsvf namespace: kserve-ci-e2e-test resourceVersion: "14077" uid: fe91929d-2d86-4a10-9dc9-8ffb2a26434f kind: Event lastTimestamp: "2026-05-25T13:14:45Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.281s (2.281s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-05-25T13:14:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:14:45Z" name: error-404-isvc-889c1-predictor-b649dc755-pwsvf.18b2d1425489f32b namespace: kserve-ci-e2e-test resourceVersion: "14317" uid: 6d422612-3d1a-47c8-bb38-23fc6f56e69f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-889c1-predictor-b649dc755-pwsvf namespace: kserve-ci-e2e-test resourceVersion: "14077" uid: fe91929d-2d86-4a10-9dc9-8ffb2a26434f kind: Event lastTimestamp: "2026-05-25T13:14:45Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T13:14:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:14:45Z" name: error-404-isvc-889c1-predictor-b649dc755-pwsvf.18b2d14257fbca7b namespace: kserve-ci-e2e-test resourceVersion: "14318" uid: 1e1d85a4-2b8f-412b-9e16-f73e9e18e9fa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-889c1-predictor-b649dc755-pwsvf namespace: kserve-ci-e2e-test resourceVersion: "14077" uid: fe91929d-2d86-4a10-9dc9-8ffb2a26434f kind: Event lastTimestamp: "2026-05-25T13:14:45Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T13:14:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:14:45Z" name: error-404-isvc-889c1-predictor-b649dc755-pwsvf.18b2d1425872fd92 namespace: kserve-ci-e2e-test resourceVersion: "14319" uid: 72be229e-e234-42af-b04f-d46297814658 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-05-25T13:14:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-889c1-predictor-b649dc755-pwsvf namespace: kserve-ci-e2e-test resourceVersion: "14077" uid: fe91929d-2d86-4a10-9dc9-8ffb2a26434f kind: Event lastTimestamp: "2026-05-25T13:16:03Z" message: 'Readiness probe failed: dial tcp 10.132.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T13:14:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:16:03Z" name: error-404-isvc-889c1-predictor-b649dc755-pwsvf.18b2d142cb606a61 namespace: kserve-ci-e2e-test resourceVersion: "15211" uid: 1f5ce234-714c-4e71-bec0-ba2952d878fe reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-889c1-predictor-b649dc755-pwsvf namespace: kserve-ci-e2e-test resourceVersion: "14077" uid: fe91929d-2d86-4a10-9dc9-8ffb2a26434f kind: Event lastTimestamp: "2026-05-25T13:16:00Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T13:16:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:16:00Z" name: error-404-isvc-889c1-predictor-b649dc755-pwsvf.18b2d153dd9714ce namespace: kserve-ci-e2e-test resourceVersion: "15078" uid: fd988545-8eed-4bbd-88ea-8b2f39ba41ed reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-889c1-predictor-b649dc755-pwsvf namespace: kserve-ci-e2e-test resourceVersion: "14077" uid: fe91929d-2d86-4a10-9dc9-8ffb2a26434f kind: Event lastTimestamp: "2026-05-25T13:16:00Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T13:16:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:16:00Z" name: error-404-isvc-889c1-predictor-b649dc755-pwsvf.18b2d153dd983fb4 namespace: kserve-ci-e2e-test resourceVersion: "15079" uid: f8b93046-112b-45dc-8be8-5a3de34baf6f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-889c1-predictor-b649dc755-pwsvf namespace: kserve-ci-e2e-test resourceVersion: "14077" uid: fe91929d-2d86-4a10-9dc9-8ffb2a26434f kind: Event lastTimestamp: "2026-05-25T13:16:03Z" message: 'Readiness probe failed: Get "https://10.132.0.27:8643/healthz": dial tcp 10.132.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-25T13:16:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:16:03Z" name: error-404-isvc-889c1-predictor-b649dc755-pwsvf.18b2d1547d6fed11 namespace: kserve-ci-e2e-test resourceVersion: "15210" uid: 62abf6d6-ddb7-45bb-b14f-8a60a265d008 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-889c1-predictor-b649dc755 namespace: kserve-ci-e2e-test resourceVersion: "14064" uid: a85bbdc1-8a3d-4cb1-a2ef-f12b4b4e7e0f kind: Event lastTimestamp: "2026-05-25T13:14:31Z" message: 'Created pod: error-404-isvc-889c1-predictor-b649dc755-pwsvf' metadata: creationTimestamp: "2026-05-25T13:14:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:14:31Z" name: error-404-isvc-889c1-predictor-b649dc755.18b2d13ef19928c0 namespace: kserve-ci-e2e-test resourceVersion: "14076" uid: ae2fa5f5-81e9-4dd6-af46-627fff8dbb5e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-889c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14060" uid: ce74d560-2180-42b1-9d8f-f57ae5a405e4 kind: Event lastTimestamp: "2026-05-25T13:14:30Z" message: Scaled up replica set error-404-isvc-889c1-predictor-b649dc755 from 0 to 1 metadata: creationTimestamp: "2026-05-25T13:14:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:14:30Z" name: error-404-isvc-889c1-predictor.18b2d13eeef63be7 namespace: kserve-ci-e2e-test resourceVersion: "14067" uid: f1bd827c-9dff-4bd8-b926-2c7f73b2c1e5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T13:14:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-889c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14061" uid: 5d54e4ae-1a0d-4e0b-bd75-22448d3d7ea6 kind: Event lastTimestamp: "2026-05-25T13:15:16Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T13:14:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:15:16Z" name: error-404-isvc-889c1-predictor.18b2d1426d71de25 namespace: kserve-ci-e2e-test resourceVersion: "14612" uid: f7d6b9c6-c3bf-4096-b4b3-569383b5daad reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T13:14:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-889c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14061" uid: 5d54e4ae-1a0d-4e0b-bd75-22448d3d7ea6 kind: Event lastTimestamp: "2026-05-25T13:15:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T13:14:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:15:16Z" name: error-404-isvc-889c1-predictor.18b2d1426df0840f namespace: kserve-ci-e2e-test resourceVersion: "14613" uid: 24613738-469c-4f57-8f64-c857affdd459 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T13:15:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-889c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14330" uid: 5d54e4ae-1a0d-4e0b-bd75-22448d3d7ea6 kind: Event lastTimestamp: "2026-05-25T13:15:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T13:15:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:15:46Z" name: error-404-isvc-889c1-predictor.18b2d14cea9b62ff namespace: kserve-ci-e2e-test resourceVersion: "14870" uid: cfa033f3-029b-423c-addf-9cc334ea7683 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T13:15:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-889c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14330" uid: 5d54e4ae-1a0d-4e0b-bd75-22448d3d7ea6 kind: Event lastTimestamp: "2026-05-25T13:15:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T13:15:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:15:46Z" name: error-404-isvc-889c1-predictor.18b2d14ceb08a064 namespace: kserve-ci-e2e-test resourceVersion: "14871" uid: b08ea915-5d90-4454-8656-6a423562368f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T13:14:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-889c1 namespace: kserve-ci-e2e-test resourceVersion: "14046" uid: 13b584dc-4162-4ba5-b444-19c5fb01ff63 kind: Event lastTimestamp: "2026-05-25T13:15:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T13:14:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:15:33Z" name: error-404-isvc-889c1.18b2d13eeeb67cac namespace: kserve-ci-e2e-test resourceVersion: "14778" uid: 4002403c-0f75-479b-93c0-5e6cb4a17433 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:15:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-889c1 namespace: kserve-ci-e2e-test resourceVersion: "14776" uid: 13b584dc-4162-4ba5-b444-19c5fb01ff63 kind: Event lastTimestamp: "2026-05-25T13:15:33Z" message: InferenceService [error-404-isvc-889c1] is Ready metadata: creationTimestamp: "2026-05-25T13:15:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:15:33Z" name: error-404-isvc-889c1.18b2d14d86030fbe namespace: kserve-ci-e2e-test resourceVersion: "14777" uid: db8d48bf-c83c-4bfe-8e30-ccc16af5d8d0 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T13:25:46.070991Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-9ee57-predictor-d974ddf95-6jxwb namespace: kserve-ci-e2e-test resourceVersion: "19884" uid: ce196080-f42b-4918-b6f6-f42da3537e67 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-9ee57-predictor-d974ddf95-6jxwb to ip-10-0-139-18.ec2.internal metadata: creationTimestamp: "2026-05-25T13:25:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T13:25:46Z" name: error-404-isvc-9ee57-predictor-d974ddf95-6jxwb.18b2d1dc1ccd7635 namespace: kserve-ci-e2e-test resourceVersion: "19888" uid: 785111f2-a3e4-40e3-9735-00df79f75335 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c46b5f76-4gnkb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:25:46Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-9ee57-predictor-d974ddf95-6jxwb namespace: kserve-ci-e2e-test resourceVersion: "19886" uid: ce196080-f42b-4918-b6f6-f42da3537e67 kind: Event lastTimestamp: "2026-05-25T13:25:46Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-9ee57-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-25T13:25:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:25:46Z" name: error-404-isvc-9ee57-predictor-d974ddf95-6jxwb.18b2d1dc27a3f3a9 namespace: kserve-ci-e2e-test resourceVersion: "19895" uid: 29ab7882-93cd-4858-823b-f325346e94d2 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:25:47Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-9ee57-predictor-d974ddf95-6jxwb namespace: kserve-ci-e2e-test resourceVersion: "19894" uid: ce196080-f42b-4918-b6f6-f42da3537e67 kind: Event lastTimestamp: "2026-05-25T13:25:47Z" message: Add eth0 [10.132.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T13:25:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T13:25:47Z" name: error-404-isvc-9ee57-predictor-d974ddf95-6jxwb.18b2d1dc5ab22944 namespace: kserve-ci-e2e-test resourceVersion: "19922" uid: 9bc76e54-927b-487d-8f41-871cad152dac reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:25:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9ee57-predictor-d974ddf95-6jxwb namespace: kserve-ci-e2e-test resourceVersion: "19886" uid: ce196080-f42b-4918-b6f6-f42da3537e67 kind: Event lastTimestamp: "2026-05-25T13:25:47Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-25T13:25:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:25:47Z" name: error-404-isvc-9ee57-predictor-d974ddf95-6jxwb.18b2d1dc5c46436c namespace: kserve-ci-e2e-test resourceVersion: "19924" uid: c286c678-c5f9-452b-a8a6-4a347d883725 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:25:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9ee57-predictor-d974ddf95-6jxwb namespace: kserve-ci-e2e-test resourceVersion: "19886" uid: ce196080-f42b-4918-b6f6-f42da3537e67 kind: Event lastTimestamp: "2026-05-25T13:25:47Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T13:25:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:25:47Z" name: error-404-isvc-9ee57-predictor-d974ddf95-6jxwb.18b2d1dc5fc00616 namespace: kserve-ci-e2e-test resourceVersion: "19925" uid: baa0e2c6-81d5-4fd0-a4bc-6a64eb4d4d47 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:25:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9ee57-predictor-d974ddf95-6jxwb namespace: kserve-ci-e2e-test resourceVersion: "19886" uid: ce196080-f42b-4918-b6f6-f42da3537e67 kind: Event lastTimestamp: "2026-05-25T13:25:47Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T13:25:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:25:47Z" name: error-404-isvc-9ee57-predictor-d974ddf95-6jxwb.18b2d1dc6037b212 namespace: kserve-ci-e2e-test resourceVersion: "19926" uid: 423a3a07-09e3-40c9-946a-ba4ecdadd42e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:25:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9ee57-predictor-d974ddf95-6jxwb namespace: kserve-ci-e2e-test resourceVersion: "19886" uid: ce196080-f42b-4918-b6f6-f42da3537e67 kind: Event lastTimestamp: "2026-05-25T13:25:47Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-25T13:25:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:25:47Z" name: error-404-isvc-9ee57-predictor-d974ddf95-6jxwb.18b2d1dc60444a79 namespace: kserve-ci-e2e-test resourceVersion: "19927" uid: 40780c91-2f19-49ec-9968-9b55537c9885 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:25:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9ee57-predictor-d974ddf95-6jxwb namespace: kserve-ci-e2e-test resourceVersion: "19886" uid: ce196080-f42b-4918-b6f6-f42da3537e67 kind: Event lastTimestamp: "2026-05-25T13:25:47Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T13:25:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:25:47Z" name: error-404-isvc-9ee57-predictor-d974ddf95-6jxwb.18b2d1dc63cc44c1 namespace: kserve-ci-e2e-test resourceVersion: "19928" uid: 128c38b2-25e0-4751-9bcb-2b715e5d7357 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:25:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9ee57-predictor-d974ddf95-6jxwb namespace: kserve-ci-e2e-test resourceVersion: "19886" uid: ce196080-f42b-4918-b6f6-f42da3537e67 kind: Event lastTimestamp: "2026-05-25T13:25:47Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T13:25:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:25:47Z" name: error-404-isvc-9ee57-predictor-d974ddf95-6jxwb.18b2d1dc644f9b68 namespace: kserve-ci-e2e-test resourceVersion: "19929" uid: 06592d8e-3beb-40d2-946e-9df8869e587b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T13:25:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9ee57-predictor-d974ddf95-6jxwb namespace: kserve-ci-e2e-test resourceVersion: "19886" uid: ce196080-f42b-4918-b6f6-f42da3537e67 kind: Event lastTimestamp: "2026-05-25T13:26:24Z" message: 'Readiness probe failed: dial tcp 10.132.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T13:25:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:26:24Z" name: error-404-isvc-9ee57-predictor-d974ddf95-6jxwb.18b2d1dcc0c275ae namespace: kserve-ci-e2e-test resourceVersion: "20296" uid: 7607cb28-fedf-409d-9ab6-00cb81e7c374 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9ee57-predictor-d974ddf95-6jxwb namespace: kserve-ci-e2e-test resourceVersion: "19886" uid: ce196080-f42b-4918-b6f6-f42da3537e67 kind: Event lastTimestamp: "2026-05-25T13:26:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T13:26:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:26:56Z" name: error-404-isvc-9ee57-predictor-d974ddf95-6jxwb.18b2d1ec69f456b1 namespace: kserve-ci-e2e-test resourceVersion: "20777" uid: 0e486df8-0560-418c-b17f-86502840a3fd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9ee57-predictor-d974ddf95-6jxwb namespace: kserve-ci-e2e-test resourceVersion: "19886" uid: ce196080-f42b-4918-b6f6-f42da3537e67 kind: Event lastTimestamp: "2026-05-25T13:26:56Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T13:26:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:26:56Z" name: error-404-isvc-9ee57-predictor-d974ddf95-6jxwb.18b2d1ec69f58908 namespace: kserve-ci-e2e-test resourceVersion: "20779" uid: bc8febe9-598d-42a0-81f9-b123af2e0b4e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:25:46Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-9ee57-predictor-d974ddf95 namespace: kserve-ci-e2e-test resourceVersion: "19876" uid: 1fe8ab2a-33c1-410d-a35d-e1e046b61bdb kind: Event lastTimestamp: "2026-05-25T13:25:46Z" message: 'Created pod: error-404-isvc-9ee57-predictor-d974ddf95-6jxwb' metadata: creationTimestamp: "2026-05-25T13:25:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:25:46Z" name: error-404-isvc-9ee57-predictor-d974ddf95.18b2d1dc1c63bb92 namespace: kserve-ci-e2e-test resourceVersion: "19885" uid: bb11fb50-ba78-45e0-99dd-0aeaa3fe81f8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:25:46Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-9ee57-predictor namespace: kserve-ci-e2e-test resourceVersion: "19874" uid: c16ff261-7a30-4dbf-8b04-48861b0c7897 kind: Event lastTimestamp: "2026-05-25T13:25:46Z" message: Scaled up replica set error-404-isvc-9ee57-predictor-d974ddf95 from 0 to 1 metadata: creationTimestamp: "2026-05-25T13:25:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:25:46Z" name: error-404-isvc-9ee57-predictor.18b2d1dc1a8abe27 namespace: kserve-ci-e2e-test resourceVersion: "19878" uid: eaf93630-006c-404e-a185-276b2a7bdc51 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T13:26:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9ee57-predictor namespace: kserve-ci-e2e-test resourceVersion: "19875" uid: 001c2134-0ba2-4874-a212-54bd25a7cba1 kind: Event lastTimestamp: "2026-05-25T13:26:16Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T13:26:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:26:16Z" name: error-404-isvc-9ee57-predictor.18b2d1df98eb6d2e namespace: kserve-ci-e2e-test resourceVersion: "20222" uid: 12a50d45-d3af-4fd6-9f6d-d0bb411bdf2e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T13:26:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9ee57-predictor namespace: kserve-ci-e2e-test resourceVersion: "19875" uid: 001c2134-0ba2-4874-a212-54bd25a7cba1 kind: Event lastTimestamp: "2026-05-25T13:26:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T13:26:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:26:16Z" name: error-404-isvc-9ee57-predictor.18b2d1df994dadb6 namespace: kserve-ci-e2e-test resourceVersion: "20223" uid: 6b75499c-931d-4b7f-a276-f972b82642b4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T13:26:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9ee57-predictor namespace: kserve-ci-e2e-test resourceVersion: "20055" uid: 001c2134-0ba2-4874-a212-54bd25a7cba1 kind: Event lastTimestamp: "2026-05-25T13:26:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T13:26:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:26:46Z" name: error-404-isvc-9ee57-predictor.18b2d1e696b217b8 namespace: kserve-ci-e2e-test resourceVersion: "20659" uid: 9674abcd-2dec-451f-816b-5cc9e8e7bfe2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T13:26:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9ee57-predictor namespace: kserve-ci-e2e-test resourceVersion: "20055" uid: 001c2134-0ba2-4874-a212-54bd25a7cba1 kind: Event lastTimestamp: "2026-05-25T13:26:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T13:26:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:26:46Z" name: error-404-isvc-9ee57-predictor.18b2d1e6971ad169 namespace: kserve-ci-e2e-test resourceVersion: "20660" uid: acb6316a-21d7-4a48-903d-6b074657300f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T13:25:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9ee57 namespace: kserve-ci-e2e-test resourceVersion: "19856" uid: bd0ed480-11bf-4094-b64a-8f496347672a kind: Event lastTimestamp: "2026-05-25T13:26:34Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T13:25:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:26:34Z" name: error-404-isvc-9ee57.18b2d1dc1a40cb72 namespace: kserve-ci-e2e-test resourceVersion: "20541" uid: ceef20ff-0aa4-4b22-ae6b-fc4dd02a0ce7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:25:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9ee57 namespace: kserve-ci-e2e-test resourceVersion: "19856" uid: bd0ed480-11bf-4094-b64a-8f496347672a kind: Event lastTimestamp: "2026-05-25T13:25:46Z" message: 'Failed to update status for InferenceService "error-404-isvc-9ee57": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-9ee57": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T13:25:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:25:46Z" name: error-404-isvc-9ee57.18b2d1dc1ab8cac6 namespace: kserve-ci-e2e-test resourceVersion: "19880" uid: ec40da5c-e534-4d3e-88d7-f8753e436ace reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:25:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9ee57 namespace: kserve-ci-e2e-test resourceVersion: "19856" uid: bd0ed480-11bf-4094-b64a-8f496347672a kind: Event lastTimestamp: "2026-05-25T13:25:46Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-9ee57": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T13:25:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:25:46Z" name: error-404-isvc-9ee57.18b2d1dc1ab92bb6 namespace: kserve-ci-e2e-test resourceVersion: "19882" uid: 209984ed-cc00-4c5f-b21a-478cac5c895b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9ee57 namespace: kserve-ci-e2e-test resourceVersion: "20539" uid: bd0ed480-11bf-4094-b64a-8f496347672a kind: Event lastTimestamp: "2026-05-25T13:26:34Z" message: InferenceService [error-404-isvc-9ee57] is Ready metadata: creationTimestamp: "2026-05-25T13:26:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:26:34Z" name: error-404-isvc-9ee57.18b2d1e77bb7665c namespace: kserve-ci-e2e-test resourceVersion: "20540" uid: 61739fe7-3d92-40bd-b7af-9911ab2a9207 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T13:25:15.916641Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-db114-predictor-7d7bc84957-tkjp8 namespace: kserve-ci-e2e-test resourceVersion: "19509" uid: de4f786e-ee21-4acf-a660-ec364cf8ac74 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-db114-predictor-7d7bc84957-tkjp8 to ip-10-0-139-18.ec2.internal metadata: creationTimestamp: "2026-05-25T13:25:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T13:25:15Z" name: error-404-isvc-db114-predictor-7d7bc84957-tkjp8.18b2d1d5177688cc namespace: kserve-ci-e2e-test resourceVersion: "19513" uid: edb039cd-76d7-46f6-b0da-53623a2a8458 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c46b5f76-4gnkb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:25:16Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-db114-predictor-7d7bc84957-tkjp8 namespace: kserve-ci-e2e-test resourceVersion: "19511" uid: de4f786e-ee21-4acf-a660-ec364cf8ac74 kind: Event lastTimestamp: "2026-05-25T13:25:16Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-db114-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-25T13:25:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:25:16Z" name: error-404-isvc-db114-predictor-7d7bc84957-tkjp8.18b2d1d521ec6076 namespace: kserve-ci-e2e-test resourceVersion: "19520" uid: b38777ae-cee4-4b33-a356-9c0a022b9b85 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:25:16Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-db114-predictor-7d7bc84957-tkjp8 namespace: kserve-ci-e2e-test resourceVersion: "19519" uid: de4f786e-ee21-4acf-a660-ec364cf8ac74 kind: Event lastTimestamp: "2026-05-25T13:25:16Z" message: Add eth0 [10.132.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T13:25:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T13:25:16Z" name: error-404-isvc-db114-predictor-7d7bc84957-tkjp8.18b2d1d5551dcbca namespace: kserve-ci-e2e-test resourceVersion: "19546" uid: a9ccd067-202d-4f10-824c-585bed736bfa reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:25:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-db114-predictor-7d7bc84957-tkjp8 namespace: kserve-ci-e2e-test resourceVersion: "19511" uid: de4f786e-ee21-4acf-a660-ec364cf8ac74 kind: Event lastTimestamp: "2026-05-25T13:25:16Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-25T13:25:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:25:16Z" name: error-404-isvc-db114-predictor-7d7bc84957-tkjp8.18b2d1d556611bc0 namespace: kserve-ci-e2e-test resourceVersion: "19549" uid: b4b4310a-0e6d-4797-837a-5d1d2fc4a08d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:25:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-db114-predictor-7d7bc84957-tkjp8 namespace: kserve-ci-e2e-test resourceVersion: "19511" uid: de4f786e-ee21-4acf-a660-ec364cf8ac74 kind: Event lastTimestamp: "2026-05-25T13:25:17Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T13:25:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:25:17Z" name: error-404-isvc-db114-predictor-7d7bc84957-tkjp8.18b2d1d559c33426 namespace: kserve-ci-e2e-test resourceVersion: "19550" uid: cc6900bc-8a51-439d-9443-bf45d050e0dc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:25:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-db114-predictor-7d7bc84957-tkjp8 namespace: kserve-ci-e2e-test resourceVersion: "19511" uid: de4f786e-ee21-4acf-a660-ec364cf8ac74 kind: Event lastTimestamp: "2026-05-25T13:25:17Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T13:25:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:25:17Z" name: error-404-isvc-db114-predictor-7d7bc84957-tkjp8.18b2d1d55a399922 namespace: kserve-ci-e2e-test resourceVersion: "19551" uid: 0807b314-022a-4cfd-9437-cf7083adcd7e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:25:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-db114-predictor-7d7bc84957-tkjp8 namespace: kserve-ci-e2e-test resourceVersion: "19511" uid: de4f786e-ee21-4acf-a660-ec364cf8ac74 kind: Event lastTimestamp: "2026-05-25T13:25:17Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-25T13:25:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:25:17Z" name: error-404-isvc-db114-predictor-7d7bc84957-tkjp8.18b2d1d55a4774d7 namespace: kserve-ci-e2e-test resourceVersion: "19552" uid: 97c200c6-edd2-4fa2-8737-ea88c4ed21cb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:25:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-db114-predictor-7d7bc84957-tkjp8 namespace: kserve-ci-e2e-test resourceVersion: "19511" uid: de4f786e-ee21-4acf-a660-ec364cf8ac74 kind: Event lastTimestamp: "2026-05-25T13:25:17Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T13:25:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:25:17Z" name: error-404-isvc-db114-predictor-7d7bc84957-tkjp8.18b2d1d55dc18e47 namespace: kserve-ci-e2e-test resourceVersion: "19554" uid: 7be9d74d-dab8-44f1-b9a8-3e3190c89b59 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:25:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-db114-predictor-7d7bc84957-tkjp8 namespace: kserve-ci-e2e-test resourceVersion: "19511" uid: de4f786e-ee21-4acf-a660-ec364cf8ac74 kind: Event lastTimestamp: "2026-05-25T13:25:17Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T13:25:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:25:17Z" name: error-404-isvc-db114-predictor-7d7bc84957-tkjp8.18b2d1d55e39e4bc namespace: kserve-ci-e2e-test resourceVersion: "19555" uid: 5a645a0f-34f1-4db7-be4c-0299560fd9ef reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-05-25T13:25:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-db114-predictor-7d7bc84957-tkjp8 namespace: kserve-ci-e2e-test resourceVersion: "19511" uid: de4f786e-ee21-4acf-a660-ec364cf8ac74 kind: Event lastTimestamp: "2026-05-25T13:26:03Z" message: 'Readiness probe failed: dial tcp 10.132.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T13:25:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:26:03Z" name: error-404-isvc-db114-predictor-7d7bc84957-tkjp8.18b2d1d581f95c19 namespace: kserve-ci-e2e-test resourceVersion: "20087" uid: 51a8f748-b049-4845-b145-ecbf6d652770 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-db114-predictor-7d7bc84957-tkjp8 namespace: kserve-ci-e2e-test resourceVersion: "19511" uid: de4f786e-ee21-4acf-a660-ec364cf8ac74 kind: Event lastTimestamp: "2026-05-25T13:26:26Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T13:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:26:26Z" name: error-404-isvc-db114-predictor-7d7bc84957-tkjp8.18b2d1e57638e502 namespace: kserve-ci-e2e-test resourceVersion: "20384" uid: 5e867148-d52d-474b-9279-4b1bd33bbfcf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-db114-predictor-7d7bc84957-tkjp8 namespace: kserve-ci-e2e-test resourceVersion: "19511" uid: de4f786e-ee21-4acf-a660-ec364cf8ac74 kind: Event lastTimestamp: "2026-05-25T13:26:26Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T13:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:26:26Z" name: error-404-isvc-db114-predictor-7d7bc84957-tkjp8.18b2d1e5763960d0 namespace: kserve-ci-e2e-test resourceVersion: "20388" uid: 5e233e3b-be49-471f-961d-f4f62a8079bd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-db114-predictor-7d7bc84957-tkjp8 namespace: kserve-ci-e2e-test resourceVersion: "19511" uid: de4f786e-ee21-4acf-a660-ec364cf8ac74 kind: Event lastTimestamp: "2026-05-25T13:26:28Z" message: 'Readiness probe failed: Get "https://10.132.0.34:8643/healthz": dial tcp 10.132.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-25T13:26:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:26:28Z" name: error-404-isvc-db114-predictor-7d7bc84957-tkjp8.18b2d1e60a09e887 namespace: kserve-ci-e2e-test resourceVersion: "20462" uid: e8f654f2-926b-458f-9861-4602687d03d3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:25:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-db114-predictor-7d7bc84957 namespace: kserve-ci-e2e-test resourceVersion: "19501" uid: 2a25f020-acbb-4233-a5c3-03943882e5b1 kind: Event lastTimestamp: "2026-05-25T13:25:15Z" message: 'Created pod: error-404-isvc-db114-predictor-7d7bc84957-tkjp8' metadata: creationTimestamp: "2026-05-25T13:25:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:25:15Z" name: error-404-isvc-db114-predictor-7d7bc84957.18b2d1d517107b30 namespace: kserve-ci-e2e-test resourceVersion: "19510" uid: 5adde6dc-c556-4c02-a12e-cec03d1b0d82 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:25:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-db114-predictor namespace: kserve-ci-e2e-test resourceVersion: "19496" uid: a2ddd145-1f22-447d-8c03-49204a9280f3 kind: Event lastTimestamp: "2026-05-25T13:25:15Z" message: Scaled up replica set error-404-isvc-db114-predictor-7d7bc84957 from 0 to 1 metadata: creationTimestamp: "2026-05-25T13:25:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:25:15Z" name: error-404-isvc-db114-predictor.18b2d1d51594eb5e namespace: kserve-ci-e2e-test resourceVersion: "19504" uid: 80985645-550c-4cd9-83e9-e467d4a6665a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T13:25:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-db114-predictor namespace: kserve-ci-e2e-test resourceVersion: "19500" uid: 0b9728c5-cfa8-475b-b3ed-097b970bedcc kind: Event lastTimestamp: "2026-05-25T13:25:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T13:25:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:25:45Z" name: error-404-isvc-db114-predictor.18b2d1d89464a00b namespace: kserve-ci-e2e-test resourceVersion: "19861" uid: 970bc275-3310-487c-b8bf-7c72866c9722 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T13:25:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-db114-predictor namespace: kserve-ci-e2e-test resourceVersion: "19500" uid: 0b9728c5-cfa8-475b-b3ed-097b970bedcc kind: Event lastTimestamp: "2026-05-25T13:25:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T13:25:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:25:45Z" name: error-404-isvc-db114-predictor.18b2d1d894d85df5 namespace: kserve-ci-e2e-test resourceVersion: "19864" uid: ad9031f0-4251-4857-93a8-d38bf367816b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T13:26:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-db114-predictor namespace: kserve-ci-e2e-test resourceVersion: "19672" uid: 0b9728c5-cfa8-475b-b3ed-097b970bedcc kind: Event lastTimestamp: "2026-05-25T13:26:15Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T13:26:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:26:15Z" name: error-404-isvc-db114-predictor.18b2d1df9259d8f1 namespace: kserve-ci-e2e-test resourceVersion: "20211" uid: afc0be86-a031-43e4-8360-6a3820677de6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T13:26:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-db114-predictor namespace: kserve-ci-e2e-test resourceVersion: "19672" uid: 0b9728c5-cfa8-475b-b3ed-097b970bedcc kind: Event lastTimestamp: "2026-05-25T13:26:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T13:26:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:26:15Z" name: error-404-isvc-db114-predictor.18b2d1df92b754db namespace: kserve-ci-e2e-test resourceVersion: "20212" uid: 63d91672-22c7-46d3-830f-44a0c799fc23 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T13:25:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-db114 namespace: kserve-ci-e2e-test resourceVersion: "19479" uid: 2fbd7204-7b08-461c-adb1-3da97964a669 kind: Event lastTimestamp: "2026-05-25T13:26:13Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T13:25:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:26:13Z" name: error-404-isvc-db114.18b2d1d5156efa00 namespace: kserve-ci-e2e-test resourceVersion: "20168" uid: e0c3ac90-3211-4fef-b5d3-10a50356e4ea reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:25:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-db114 namespace: kserve-ci-e2e-test resourceVersion: "19479" uid: 2fbd7204-7b08-461c-adb1-3da97964a669 kind: Event lastTimestamp: "2026-05-25T13:25:15Z" message: 'Failed to update status for InferenceService "error-404-isvc-db114": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-db114": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T13:25:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:25:15Z" name: error-404-isvc-db114.18b2d1d515e73713 namespace: kserve-ci-e2e-test resourceVersion: "19506" uid: b35145bd-8776-4cfd-878b-0f98485db488 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:25:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-db114 namespace: kserve-ci-e2e-test resourceVersion: "19479" uid: 2fbd7204-7b08-461c-adb1-3da97964a669 kind: Event lastTimestamp: "2026-05-25T13:25:15Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-db114": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T13:25:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:25:15Z" name: error-404-isvc-db114.18b2d1d515e78d84 namespace: kserve-ci-e2e-test resourceVersion: "19507" uid: 58d5c1a2-b34b-4dc3-8610-9730983185f6 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-db114 namespace: kserve-ci-e2e-test resourceVersion: "20166" uid: 2fbd7204-7b08-461c-adb1-3da97964a669 kind: Event lastTimestamp: "2026-05-25T13:26:13Z" message: InferenceService [error-404-isvc-db114] is Ready metadata: creationTimestamp: "2026-05-25T13:26:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:26:13Z" name: error-404-isvc-db114.18b2d1e290d7d79e namespace: kserve-ci-e2e-test resourceVersion: "20167" uid: 89ab5029-d85a-4d83-875b-7c88e1e08970 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T13:14:30.960618Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-c6nfj namespace: kserve-ci-e2e-test resourceVersion: "14048" uid: 9e671986-36b9-4c8b-bc33-12613fa1f8d9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-678784f46b-c6nfj to ip-10-0-135-103.ec2.internal metadata: creationTimestamp: "2026-05-25T13:14:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T13:14:30Z" name: isvc-sklearn-graph-1-predictor-678784f46b-c6nfj.18b2d13eed16a389 namespace: kserve-ci-e2e-test resourceVersion: "14055" uid: e357eb5b-54fd-49d7-afb0-bc059fa0eaa6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c46b5f76-4gnkb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:31Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-c6nfj namespace: kserve-ci-e2e-test resourceVersion: "14063" uid: 9e671986-36b9-4c8b-bc33-12613fa1f8d9 kind: Event lastTimestamp: "2026-05-25T13:14:31Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T13:14:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T13:14:31Z" name: isvc-sklearn-graph-1-predictor-678784f46b-c6nfj.18b2d13f082f8476 namespace: kserve-ci-e2e-test resourceVersion: "14118" uid: ba64c90e-8b58-4f8c-8914-f20fe9dc152d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-c6nfj namespace: kserve-ci-e2e-test resourceVersion: "14050" uid: 9e671986-36b9-4c8b-bc33-12613fa1f8d9 kind: Event lastTimestamp: "2026-05-25T13:14:31Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:30bffa55f5e3e0e2f6d023c0f84846c206091122d724f5a0e810b4fb957dd389" metadata: creationTimestamp: "2026-05-25T13:14:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:14:31Z" name: isvc-sklearn-graph-1-predictor-678784f46b-c6nfj.18b2d13f09b2f7d1 namespace: kserve-ci-e2e-test resourceVersion: "14120" uid: 524cdf88-9f3f-4014-8dcf-898659d02a56 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:36Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-c6nfj namespace: kserve-ci-e2e-test resourceVersion: "14050" uid: 9e671986-36b9-4c8b-bc33-12613fa1f8d9 kind: Event lastTimestamp: "2026-05-25T13:14:36Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:30bffa55f5e3e0e2f6d023c0f84846c206091122d724f5a0e810b4fb957dd389" in 4.617s (4.617s including waiting). Image size: 303109559 bytes.' metadata: creationTimestamp: "2026-05-25T13:14:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:14:36Z" name: isvc-sklearn-graph-1-predictor-678784f46b-c6nfj.18b2d1401ce906bf namespace: kserve-ci-e2e-test resourceVersion: "14213" uid: 0b9a8038-8cd4-44b8-bf4b-4974b4b172b7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:36Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-c6nfj namespace: kserve-ci-e2e-test resourceVersion: "14050" uid: 9e671986-36b9-4c8b-bc33-12613fa1f8d9 kind: Event lastTimestamp: "2026-05-25T13:14:36Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-25T13:14:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:14:36Z" name: isvc-sklearn-graph-1-predictor-678784f46b-c6nfj.18b2d140212f3e05 namespace: kserve-ci-e2e-test resourceVersion: "14214" uid: 88c3daac-58eb-42b8-9d9e-28c82b693ba9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:36Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-c6nfj namespace: kserve-ci-e2e-test resourceVersion: "14050" uid: 9e671986-36b9-4c8b-bc33-12613fa1f8d9 kind: Event lastTimestamp: "2026-05-25T13:14:36Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-25T13:14:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:14:36Z" name: isvc-sklearn-graph-1-predictor-678784f46b-c6nfj.18b2d14021cb8c87 namespace: kserve-ci-e2e-test resourceVersion: "14216" uid: e6648f9b-7e94-4623-bf12-ad4916b617bd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-c6nfj namespace: kserve-ci-e2e-test resourceVersion: "14050" uid: 9e671986-36b9-4c8b-bc33-12613fa1f8d9 kind: Event lastTimestamp: "2026-05-25T13:14:40Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1532" metadata: creationTimestamp: "2026-05-25T13:14:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:14:40Z" name: isvc-sklearn-graph-1-predictor-678784f46b-c6nfj.18b2d1411db1eb43 namespace: kserve-ci-e2e-test resourceVersion: "14264" uid: b2da8f1e-6c8b-45c3-9a56-3c052d0dceec reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-c6nfj namespace: kserve-ci-e2e-test resourceVersion: "14050" uid: 9e671986-36b9-4c8b-bc33-12613fa1f8d9 kind: Event lastTimestamp: "2026-05-25T13:14:49Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1532" in 9.546s (9.546s including waiting). Image size: 1560933302 bytes.' metadata: creationTimestamp: "2026-05-25T13:14:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:14:49Z" name: isvc-sklearn-graph-1-predictor-678784f46b-c6nfj.18b2d14356bbe0cb namespace: kserve-ci-e2e-test resourceVersion: "14378" uid: fb208a48-6918-4891-85de-a4ad3665e636 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-c6nfj namespace: kserve-ci-e2e-test resourceVersion: "14050" uid: 9e671986-36b9-4c8b-bc33-12613fa1f8d9 kind: Event lastTimestamp: "2026-05-25T13:14:49Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T13:14:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:14:49Z" name: isvc-sklearn-graph-1-predictor-678784f46b-c6nfj.18b2d1435ae111ab namespace: kserve-ci-e2e-test resourceVersion: "14381" uid: cb8de0be-76ae-40bd-9807-bcdd408db335 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-c6nfj namespace: kserve-ci-e2e-test resourceVersion: "14050" uid: 9e671986-36b9-4c8b-bc33-12613fa1f8d9 kind: Event lastTimestamp: "2026-05-25T13:14:49Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T13:14:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:14:50Z" name: isvc-sklearn-graph-1-predictor-678784f46b-c6nfj.18b2d1435b70702f namespace: kserve-ci-e2e-test resourceVersion: "14385" uid: 64971e7b-fa32-4e7d-9c39-454ad1d080d6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-c6nfj namespace: kserve-ci-e2e-test resourceVersion: "14050" uid: 9e671986-36b9-4c8b-bc33-12613fa1f8d9 kind: Event lastTimestamp: "2026-05-25T13:14:49Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-25T13:14:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:14:50Z" name: isvc-sklearn-graph-1-predictor-678784f46b-c6nfj.18b2d1435b889b2a namespace: kserve-ci-e2e-test resourceVersion: "14386" uid: f85f412c-45d4-40ca-8a37-ba83837a2a55 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-c6nfj namespace: kserve-ci-e2e-test resourceVersion: "14050" uid: 9e671986-36b9-4c8b-bc33-12613fa1f8d9 kind: Event lastTimestamp: "2026-05-25T13:14:50Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T13:14:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:14:50Z" name: isvc-sklearn-graph-1-predictor-678784f46b-c6nfj.18b2d1435f6fca1b namespace: kserve-ci-e2e-test resourceVersion: "14388" uid: ac8b9715-8cf4-4b3d-bbd7-1233161b21c9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-c6nfj namespace: kserve-ci-e2e-test resourceVersion: "14050" uid: 9e671986-36b9-4c8b-bc33-12613fa1f8d9 kind: Event lastTimestamp: "2026-05-25T13:14:50Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T13:14:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:14:50Z" name: isvc-sklearn-graph-1-predictor-678784f46b-c6nfj.18b2d1435ffaf62f namespace: kserve-ci-e2e-test resourceVersion: "14389" uid: fa738ba1-adf6-448d-9a1f-df93f61cd743 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-05-25T13:14:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-c6nfj namespace: kserve-ci-e2e-test resourceVersion: "14050" uid: 9e671986-36b9-4c8b-bc33-12613fa1f8d9 kind: Event lastTimestamp: "2026-05-25T13:15:46Z" message: 'Readiness probe failed: dial tcp 10.134.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T13:14:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:15:46Z" name: isvc-sklearn-graph-1-predictor-678784f46b-c6nfj.18b2d1437533f2a0 namespace: kserve-ci-e2e-test resourceVersion: "14878" uid: 3dc1b191-6aea-4228-be4c-7b5e18194581 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-c6nfj namespace: kserve-ci-e2e-test resourceVersion: "14050" uid: 9e671986-36b9-4c8b-bc33-12613fa1f8d9 kind: Event lastTimestamp: "2026-05-25T13:16:31Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T13:16:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:16:31Z" name: isvc-sklearn-graph-1-predictor-678784f46b-c6nfj.18b2d15ae32155e3 namespace: kserve-ci-e2e-test resourceVersion: "15591" uid: bb79d154-5566-40c4-a03a-0497f7f0b9d8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-c6nfj namespace: kserve-ci-e2e-test resourceVersion: "14050" uid: 9e671986-36b9-4c8b-bc33-12613fa1f8d9 kind: Event lastTimestamp: "2026-05-25T13:16:31Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T13:16:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:16:31Z" name: isvc-sklearn-graph-1-predictor-678784f46b-c6nfj.18b2d15ae321bd38 namespace: kserve-ci-e2e-test resourceVersion: "15593" uid: 0cbb7fe5-8c59-42e7-ab5f-dc9d44c7131a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-c6nfj namespace: kserve-ci-e2e-test resourceVersion: "14050" uid: 9e671986-36b9-4c8b-bc33-12613fa1f8d9 kind: Event lastTimestamp: "2026-05-25T13:16:31Z" message: 'Readiness probe failed: Get "https://10.134.0.33:8643/healthz": dial tcp 10.134.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-25T13:16:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:16:31Z" name: isvc-sklearn-graph-1-predictor-678784f46b-c6nfj.18b2d15af972a2ed namespace: kserve-ci-e2e-test resourceVersion: "15630" uid: 9b2751e2-cb07-46c5-88df-cb746c672755 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-678784f46b namespace: kserve-ci-e2e-test resourceVersion: "14040" uid: a9efff22-401e-45f9-a209-f2526317298b kind: Event lastTimestamp: "2026-05-25T13:14:30Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-678784f46b-c6nfj' metadata: creationTimestamp: "2026-05-25T13:14:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:14:30Z" name: isvc-sklearn-graph-1-predictor-678784f46b.18b2d13eec635f6f namespace: kserve-ci-e2e-test resourceVersion: "14049" uid: f68bbbf3-e2f2-44dc-83de-63d326a4e741 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14038" uid: 8c9bb466-a1ba-45ff-9561-2f5cbc4fbf54 kind: Event lastTimestamp: "2026-05-25T13:14:30Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-678784f46b from 0 to 1 metadata: creationTimestamp: "2026-05-25T13:14:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:14:30Z" name: isvc-sklearn-graph-1-predictor.18b2d13ee8a6c6b6 namespace: kserve-ci-e2e-test resourceVersion: "14043" uid: aa7f3f8e-7d29-4d5a-99d3-940e23ddcd33 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T13:14:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14039" uid: 2d20c94c-10f1-44a5-982f-499ff72c17d7 kind: Event lastTimestamp: "2026-05-25T13:15:00Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T13:14:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:15:00Z" name: isvc-sklearn-graph-1-predictor.18b2d14266f5cd58 namespace: kserve-ci-e2e-test resourceVersion: "14491" uid: 34d64d15-adc5-4962-87ab-9db9bc7bd049 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T13:14:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14039" uid: 2d20c94c-10f1-44a5-982f-499ff72c17d7 kind: Event lastTimestamp: "2026-05-25T13:15:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T13:14:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:15:00Z" name: isvc-sklearn-graph-1-predictor.18b2d142677a8125 namespace: kserve-ci-e2e-test resourceVersion: "14492" uid: a5002db6-67c1-4dd8-b013-01c36c915594 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T13:15:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14324" uid: 2d20c94c-10f1-44a5-982f-499ff72c17d7 kind: Event lastTimestamp: "2026-05-25T13:16:30Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T13:15:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:16:31Z" name: isvc-sklearn-graph-1-predictor.18b2d14964e5f988 namespace: kserve-ci-e2e-test resourceVersion: "15570" uid: 08508950-199e-43ab-ab70-536f59c77999 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T13:15:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14324" uid: 2d20c94c-10f1-44a5-982f-499ff72c17d7 kind: Event lastTimestamp: "2026-05-25T13:16:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T13:15:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:16:31Z" name: isvc-sklearn-graph-1-predictor.18b2d149654a37ef namespace: kserve-ci-e2e-test resourceVersion: "15578" uid: fa9c571e-6aaf-41b2-94f2-1e668ec96a79 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-05-25T13:14:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14024" uid: 8321d498-4861-40de-984e-697eea0d838c kind: Event lastTimestamp: "2026-05-25T13:15:56Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T13:14:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:15:56Z" name: isvc-sklearn-graph-1.18b2d13ee841379d namespace: kserve-ci-e2e-test resourceVersion: "14999" uid: edf49675-ef7f-479a-815d-3caec455d8fb reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:15:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14997" uid: 8321d498-4861-40de-984e-697eea0d838c kind: Event lastTimestamp: "2026-05-25T13:15:56Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-05-25T13:15:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:15:56Z" name: isvc-sklearn-graph-1.18b2d152d886667d namespace: kserve-ci-e2e-test resourceVersion: "14998" uid: 29bb8e97-1d83-4491-9015-c58fe93108f5 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T13:14:31.909079Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-sxdzl namespace: kserve-ci-e2e-test resourceVersion: "14144" uid: 706b7f73-1a67-41ce-bd5e-0b14892b0acf kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-54c99bfc7d-sxdzl to ip-10-0-135-103.ec2.internal metadata: creationTimestamp: "2026-05-25T13:14:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T13:14:31Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-sxdzl.18b2d13f259f0162 namespace: kserve-ci-e2e-test resourceVersion: "14148" uid: 9468b9d6-129d-4221-9a24-f12234233427 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c46b5f76-4gnkb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:32Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-sxdzl namespace: kserve-ci-e2e-test resourceVersion: "14152" uid: 706b7f73-1a67-41ce-bd5e-0b14892b0acf kind: Event lastTimestamp: "2026-05-25T13:14:32Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T13:14:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T13:14:32Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-sxdzl.18b2d13f4587fb26 namespace: kserve-ci-e2e-test resourceVersion: "14162" uid: db38621c-b13a-4546-940e-2d5c953d0e0c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:33Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-sxdzl namespace: kserve-ci-e2e-test resourceVersion: "14146" uid: 706b7f73-1a67-41ce-bd5e-0b14892b0acf kind: Event lastTimestamp: "2026-05-25T13:14:33Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:30bffa55f5e3e0e2f6d023c0f84846c206091122d724f5a0e810b4fb957dd389" metadata: creationTimestamp: "2026-05-25T13:14:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:14:33Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-sxdzl.18b2d13f75d922b4 namespace: kserve-ci-e2e-test resourceVersion: "14176" uid: 55201179-ec89-4238-84af-2c915da59538 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:36Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-sxdzl namespace: kserve-ci-e2e-test resourceVersion: "14146" uid: 706b7f73-1a67-41ce-bd5e-0b14892b0acf kind: Event lastTimestamp: "2026-05-25T13:14:36Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:30bffa55f5e3e0e2f6d023c0f84846c206091122d724f5a0e810b4fb957dd389" in 2.801s (2.801s including waiting). Image size: 303109559 bytes.' metadata: creationTimestamp: "2026-05-25T13:14:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:14:36Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-sxdzl.18b2d1401cd414f7 namespace: kserve-ci-e2e-test resourceVersion: "14212" uid: 629bf6de-6e1e-4a7a-bfcb-5467218ee966 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:36Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-sxdzl namespace: kserve-ci-e2e-test resourceVersion: "14146" uid: 706b7f73-1a67-41ce-bd5e-0b14892b0acf kind: Event lastTimestamp: "2026-05-25T13:14:36Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-25T13:14:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:14:36Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-sxdzl.18b2d140213c7686 namespace: kserve-ci-e2e-test resourceVersion: "14215" uid: 0078ea2e-9d65-4afa-b58b-faa61b14014d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:36Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-sxdzl namespace: kserve-ci-e2e-test resourceVersion: "14146" uid: 706b7f73-1a67-41ce-bd5e-0b14892b0acf kind: Event lastTimestamp: "2026-05-25T13:14:36Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-25T13:14:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:14:36Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-sxdzl.18b2d14021f4a671 namespace: kserve-ci-e2e-test resourceVersion: "14217" uid: 15e2ef73-1e66-4bc3-bcd8-ccd97f03a58b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-sxdzl namespace: kserve-ci-e2e-test resourceVersion: "14146" uid: 706b7f73-1a67-41ce-bd5e-0b14892b0acf kind: Event lastTimestamp: "2026-05-25T13:14:40Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1532" metadata: creationTimestamp: "2026-05-25T13:14:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:14:40Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-sxdzl.18b2d1411d9f8373 namespace: kserve-ci-e2e-test resourceVersion: "14263" uid: b9f6303f-708f-4056-9602-4aec1f9766d1 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-sxdzl namespace: kserve-ci-e2e-test resourceVersion: "14146" uid: 706b7f73-1a67-41ce-bd5e-0b14892b0acf kind: Event lastTimestamp: "2026-05-25T13:14:49Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1532" in 9.53s (9.53s including waiting). Image size: 1560933302 bytes.' metadata: creationTimestamp: "2026-05-25T13:14:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:14:49Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-sxdzl.18b2d14355b5c9bd namespace: kserve-ci-e2e-test resourceVersion: "14377" uid: ef4bb44c-4ecb-4c41-9b58-92ec955575d4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-sxdzl namespace: kserve-ci-e2e-test resourceVersion: "14146" uid: 706b7f73-1a67-41ce-bd5e-0b14892b0acf kind: Event lastTimestamp: "2026-05-25T13:14:49Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T13:14:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:14:49Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-sxdzl.18b2d1435ac7b1a3 namespace: kserve-ci-e2e-test resourceVersion: "14380" uid: 0f3933d1-02d0-4fb8-a7fd-8c9d51c31fa9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-sxdzl namespace: kserve-ci-e2e-test resourceVersion: "14146" uid: 706b7f73-1a67-41ce-bd5e-0b14892b0acf kind: Event lastTimestamp: "2026-05-25T13:14:49Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T13:14:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:14:49Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-sxdzl.18b2d1435b478561 namespace: kserve-ci-e2e-test resourceVersion: "14382" uid: 1795e91c-77c4-46d8-869d-f0ff80f3d8d9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-sxdzl namespace: kserve-ci-e2e-test resourceVersion: "14146" uid: 706b7f73-1a67-41ce-bd5e-0b14892b0acf kind: Event lastTimestamp: "2026-05-25T13:14:49Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-25T13:14:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:14:49Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-sxdzl.18b2d1435b684cab namespace: kserve-ci-e2e-test resourceVersion: "14383" uid: 968aed76-c966-4a23-a386-18d7968e79c2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-sxdzl namespace: kserve-ci-e2e-test resourceVersion: "14146" uid: 706b7f73-1a67-41ce-bd5e-0b14892b0acf kind: Event lastTimestamp: "2026-05-25T13:14:50Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T13:14:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:14:50Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-sxdzl.18b2d1435f650635 namespace: kserve-ci-e2e-test resourceVersion: "14387" uid: 2d1fbe55-4a89-4404-9ac2-4327abed6ad7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-sxdzl namespace: kserve-ci-e2e-test resourceVersion: "14146" uid: 706b7f73-1a67-41ce-bd5e-0b14892b0acf kind: Event lastTimestamp: "2026-05-25T13:14:50Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T13:14:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:14:50Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-sxdzl.18b2d1435ffb22cc namespace: kserve-ci-e2e-test resourceVersion: "14390" uid: b5849914-2ebc-42b9-9038-4eee0f133e62 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-05-25T13:14:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-sxdzl namespace: kserve-ci-e2e-test resourceVersion: "14146" uid: 706b7f73-1a67-41ce-bd5e-0b14892b0acf kind: Event lastTimestamp: "2026-05-25T13:15:47Z" message: 'Readiness probe failed: dial tcp 10.134.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T13:14:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:15:47Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-sxdzl.18b2d143b0fbc7aa namespace: kserve-ci-e2e-test resourceVersion: "14887" uid: 2e4c45cb-2dbb-43fe-ac95-10b4bd450228 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-sxdzl namespace: kserve-ci-e2e-test resourceVersion: "14146" uid: 706b7f73-1a67-41ce-bd5e-0b14892b0acf kind: Event lastTimestamp: "2026-05-25T13:16:30Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T13:16:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:16:30Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-sxdzl.18b2d15ade877e11 namespace: kserve-ci-e2e-test resourceVersion: "15557" uid: 2214f331-f546-4d82-93e7-fe2e4eedd069 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-sxdzl namespace: kserve-ci-e2e-test resourceVersion: "14146" uid: 706b7f73-1a67-41ce-bd5e-0b14892b0acf kind: Event lastTimestamp: "2026-05-25T13:16:30Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T13:16:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:16:30Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-sxdzl.18b2d15ade8815d7 namespace: kserve-ci-e2e-test resourceVersion: "15564" uid: 1b460cfd-fa5d-47e7-8df6-83b05ac0df61 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-sxdzl namespace: kserve-ci-e2e-test resourceVersion: "14146" uid: 706b7f73-1a67-41ce-bd5e-0b14892b0acf kind: Event lastTimestamp: "2026-05-25T13:16:32Z" message: 'Readiness probe failed: Get "https://10.134.0.34:8643/healthz": dial tcp 10.134.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-25T13:16:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:16:32Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-sxdzl.18b2d15b3546d478 namespace: kserve-ci-e2e-test resourceVersion: "15661" uid: b7f6247b-03b5-4f87-9407-39ea518b8d64 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-54c99bfc7d namespace: kserve-ci-e2e-test resourceVersion: "14130" uid: 355c5ece-70cb-4059-a789-9b819377e1ca kind: Event lastTimestamp: "2026-05-25T13:14:31Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-54c99bfc7d-sxdzl' metadata: creationTimestamp: "2026-05-25T13:14:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:14:31Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d.18b2d13f2529a0ec namespace: kserve-ci-e2e-test resourceVersion: "14145" uid: ad6b80dd-a7f9-4075-92f0-9da6078fc505 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14128" uid: 06c366d3-6115-4766-a9bb-1c6ec7e4b512 kind: Event lastTimestamp: "2026-05-25T13:14:31Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-54c99bfc7d from 0 to 1 metadata: creationTimestamp: "2026-05-25T13:14:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:14:31Z" name: isvc-sklearn-graph-2-predictor.18b2d13f0eb4ca8e namespace: kserve-ci-e2e-test resourceVersion: "14131" uid: d9023fc6-ff8b-46be-aad3-de2acbbed8bf reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T13:14:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14129" uid: b53835da-a17d-462d-8830-3a9ad705221e kind: Event lastTimestamp: "2026-05-25T13:15:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T13:14:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:15:01Z" name: isvc-sklearn-graph-2-predictor.18b2d1428d3b2aee namespace: kserve-ci-e2e-test resourceVersion: "14503" uid: 786336d5-75ec-48ca-a1e7-5bd03b31ef4e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T13:14:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14129" uid: b53835da-a17d-462d-8830-3a9ad705221e kind: Event lastTimestamp: "2026-05-25T13:15:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T13:14:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:15:01Z" name: isvc-sklearn-graph-2-predictor.18b2d1428da51c59 namespace: kserve-ci-e2e-test resourceVersion: "14504" uid: 5239e0d2-3951-4a03-a47b-91e7a0dc0ba1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T13:15:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14341" uid: b53835da-a17d-462d-8830-3a9ad705221e kind: Event lastTimestamp: "2026-05-25T13:16:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T13:15:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:16:16Z" name: isvc-sklearn-graph-2-predictor.18b2d1498b1e3660 namespace: kserve-ci-e2e-test resourceVersion: "15339" uid: fdcbbbd8-9b00-4a9f-a7b7-bf3b6d3fbf6d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T13:15:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14341" uid: b53835da-a17d-462d-8830-3a9ad705221e kind: Event lastTimestamp: "2026-05-25T13:16:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T13:15:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:16:16Z" name: isvc-sklearn-graph-2-predictor.18b2d1498b8815ca namespace: kserve-ci-e2e-test resourceVersion: "15340" uid: e9ee6ee7-5588-46fd-8d03-38dc6fe5bad5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-05-25T13:14:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14102" uid: 58ef32b6-3fe4-459f-85b2-7fc98d14d078 kind: Event lastTimestamp: "2026-05-25T13:15:57Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T13:14:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:15:57Z" name: isvc-sklearn-graph-2.18b2d13f0e910bef namespace: kserve-ci-e2e-test resourceVersion: "15015" uid: 8586ff37-1460-4652-a3a5-e8ea4407e8f3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:15:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "15012" uid: 58ef32b6-3fe4-459f-85b2-7fc98d14d078 kind: Event lastTimestamp: "2026-05-25T13:15:57Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-05-25T13:15:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:15:57Z" name: isvc-sklearn-graph-2.18b2d153144c17e5 namespace: kserve-ci-e2e-test resourceVersion: "15013" uid: 2a82c064-f99f-4dee-9257-0e072c8c32a9 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T13:14:31.258509Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-tzb6t namespace: kserve-ci-e2e-test resourceVersion: "14106" uid: 105d3a7c-e7b0-46da-b3c1-3213e30d7be3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-669d8d6456-tzb6t to ip-10-0-130-213.ec2.internal metadata: creationTimestamp: "2026-05-25T13:14:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T13:14:31Z" name: isvc-xgboost-graph-predictor-669d8d6456-tzb6t.18b2d13efed800fb namespace: kserve-ci-e2e-test resourceVersion: "14110" uid: 6c4699de-14df-408a-833e-98649f6c3057 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c46b5f76-4gnkb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:31Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-tzb6t namespace: kserve-ci-e2e-test resourceVersion: "14114" uid: 105d3a7c-e7b0-46da-b3c1-3213e30d7be3 kind: Event lastTimestamp: "2026-05-25T13:14:31Z" message: Add eth0 [10.133.0.17/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T13:14:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T13:14:31Z" name: isvc-xgboost-graph-predictor-669d8d6456-tzb6t.18b2d13f184d00fe namespace: kserve-ci-e2e-test resourceVersion: "14137" uid: fe3982e1-d45a-4767-aa9b-c56e15f1450d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-tzb6t namespace: kserve-ci-e2e-test resourceVersion: "14108" uid: 105d3a7c-e7b0-46da-b3c1-3213e30d7be3 kind: Event lastTimestamp: "2026-05-25T13:14:31Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:30bffa55f5e3e0e2f6d023c0f84846c206091122d724f5a0e810b4fb957dd389" metadata: creationTimestamp: "2026-05-25T13:14:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:14:31Z" name: isvc-xgboost-graph-predictor-669d8d6456-tzb6t.18b2d13f19adcbe3 namespace: kserve-ci-e2e-test resourceVersion: "14139" uid: 57b5f61f-d069-404a-813d-5a0bc6ea77eb reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-213.ec2.internal source: component: kubelet host: ip-10-0-130-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-tzb6t namespace: kserve-ci-e2e-test resourceVersion: "14108" uid: 105d3a7c-e7b0-46da-b3c1-3213e30d7be3 kind: Event lastTimestamp: "2026-05-25T13:14:34Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:30bffa55f5e3e0e2f6d023c0f84846c206091122d724f5a0e810b4fb957dd389" in 3.285s (3.285s including waiting). Image size: 303109559 bytes.' metadata: creationTimestamp: "2026-05-25T13:14:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:14:34Z" name: isvc-xgboost-graph-predictor-669d8d6456-tzb6t.18b2d13fdd7f609e namespace: kserve-ci-e2e-test resourceVersion: "14194" uid: 68e9fefa-1280-4294-9796-f1efeff71285 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-213.ec2.internal source: component: kubelet host: ip-10-0-130-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:35Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-tzb6t namespace: kserve-ci-e2e-test resourceVersion: "14108" uid: 105d3a7c-e7b0-46da-b3c1-3213e30d7be3 kind: Event lastTimestamp: "2026-05-25T13:14:35Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-25T13:14:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:14:35Z" name: isvc-xgboost-graph-predictor-669d8d6456-tzb6t.18b2d13fe0472a9f namespace: kserve-ci-e2e-test resourceVersion: "14196" uid: f850fc05-48a8-411c-94ff-95f3e48a8d4d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-213.ec2.internal source: component: kubelet host: ip-10-0-130-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:35Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-tzb6t namespace: kserve-ci-e2e-test resourceVersion: "14108" uid: 105d3a7c-e7b0-46da-b3c1-3213e30d7be3 kind: Event lastTimestamp: "2026-05-25T13:14:35Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-25T13:14:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:14:35Z" name: isvc-xgboost-graph-predictor-669d8d6456-tzb6t.18b2d13fe0ba2f57 namespace: kserve-ci-e2e-test resourceVersion: "14197" uid: 801fecd5-3be1-4eef-a93e-2e164d91d9be reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-213.ec2.internal source: component: kubelet host: ip-10-0-130-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-tzb6t namespace: kserve-ci-e2e-test resourceVersion: "14108" uid: 105d3a7c-e7b0-46da-b3c1-3213e30d7be3 kind: Event lastTimestamp: "2026-05-25T13:14:38Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-05-25T13:14:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:14:38Z" name: isvc-xgboost-graph-predictor-669d8d6456-tzb6t.18b2d140c82f7808 namespace: kserve-ci-e2e-test resourceVersion: "14248" uid: 612862a8-a332-45d9-a958-e649bed985a2 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-213.ec2.internal source: component: kubelet host: ip-10-0-130-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-tzb6t namespace: kserve-ci-e2e-test resourceVersion: "14108" uid: 105d3a7c-e7b0-46da-b3c1-3213e30d7be3 kind: Event lastTimestamp: "2026-05-25T13:14:56Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 17.762s (17.762s including waiting). Image size: 1306319098 bytes.' metadata: creationTimestamp: "2026-05-25T13:14:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:14:56Z" name: isvc-xgboost-graph-predictor-669d8d6456-tzb6t.18b2d144eaea7fb7 namespace: kserve-ci-e2e-test resourceVersion: "14449" uid: b67dea44-3eb2-4ab3-b31a-3507fa09ce22 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-213.ec2.internal source: component: kubelet host: ip-10-0-130-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-tzb6t namespace: kserve-ci-e2e-test resourceVersion: "14108" uid: 105d3a7c-e7b0-46da-b3c1-3213e30d7be3 kind: Event lastTimestamp: "2026-05-25T13:14:56Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T13:14:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:14:56Z" name: isvc-xgboost-graph-predictor-669d8d6456-tzb6t.18b2d144ee382f8f namespace: kserve-ci-e2e-test resourceVersion: "14450" uid: 579855c0-ffa2-4a79-822a-edd28f974d61 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-213.ec2.internal source: component: kubelet host: ip-10-0-130-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-tzb6t namespace: kserve-ci-e2e-test resourceVersion: "14108" uid: 105d3a7c-e7b0-46da-b3c1-3213e30d7be3 kind: Event lastTimestamp: "2026-05-25T13:14:56Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T13:14:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:14:56Z" name: isvc-xgboost-graph-predictor-669d8d6456-tzb6t.18b2d144eea97484 namespace: kserve-ci-e2e-test resourceVersion: "14451" uid: e764e3c2-aa85-473b-a672-2e093878e4f0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-213.ec2.internal source: component: kubelet host: ip-10-0-130-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-tzb6t namespace: kserve-ci-e2e-test resourceVersion: "14108" uid: 105d3a7c-e7b0-46da-b3c1-3213e30d7be3 kind: Event lastTimestamp: "2026-05-25T13:14:56Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-05-25T13:14:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:14:56Z" name: isvc-xgboost-graph-predictor-669d8d6456-tzb6t.18b2d144eeb56cc6 namespace: kserve-ci-e2e-test resourceVersion: "14452" uid: 9d6e29fb-c0c6-462f-a265-72c30e179c1e reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-213.ec2.internal source: component: kubelet host: ip-10-0-130-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-tzb6t namespace: kserve-ci-e2e-test resourceVersion: "14108" uid: 105d3a7c-e7b0-46da-b3c1-3213e30d7be3 kind: Event lastTimestamp: "2026-05-25T13:14:58Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.143s (2.143s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-05-25T13:14:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:14:58Z" name: isvc-xgboost-graph-predictor-669d8d6456-tzb6t.18b2d1456e73c8fd namespace: kserve-ci-e2e-test resourceVersion: "14470" uid: 712860bb-ea04-48c6-8a73-3a1c8afe4dea reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-213.ec2.internal source: component: kubelet host: ip-10-0-130-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-tzb6t namespace: kserve-ci-e2e-test resourceVersion: "14108" uid: 105d3a7c-e7b0-46da-b3c1-3213e30d7be3 kind: Event lastTimestamp: "2026-05-25T13:14:58Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T13:14:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:14:58Z" name: isvc-xgboost-graph-predictor-669d8d6456-tzb6t.18b2d14571502f8a namespace: kserve-ci-e2e-test resourceVersion: "14471" uid: f3c88823-669b-47cd-b689-cc336549b340 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-213.ec2.internal source: component: kubelet host: ip-10-0-130-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-tzb6t namespace: kserve-ci-e2e-test resourceVersion: "14108" uid: 105d3a7c-e7b0-46da-b3c1-3213e30d7be3 kind: Event lastTimestamp: "2026-05-25T13:14:58Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T13:14:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:14:58Z" name: isvc-xgboost-graph-predictor-669d8d6456-tzb6t.18b2d14571c48753 namespace: kserve-ci-e2e-test resourceVersion: "14472" uid: b0dbd8f0-0e2f-441e-b70f-de5ad3493455 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-213.ec2.internal source: component: kubelet host: ip-10-0-130-213.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-05-25T13:14:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-tzb6t namespace: kserve-ci-e2e-test resourceVersion: "14108" uid: 105d3a7c-e7b0-46da-b3c1-3213e30d7be3 kind: Event lastTimestamp: "2026-05-25T13:15:55Z" message: 'Readiness probe failed: dial tcp 10.133.0.17:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T13:14:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:15:56Z" name: isvc-xgboost-graph-predictor-669d8d6456-tzb6t.18b2d145af8bbf03 namespace: kserve-ci-e2e-test resourceVersion: "14986" uid: 9e8ce4f6-469a-41b7-b65e-cf4376f7931a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-213.ec2.internal source: component: kubelet host: ip-10-0-130-213.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-tzb6t namespace: kserve-ci-e2e-test resourceVersion: "14108" uid: 105d3a7c-e7b0-46da-b3c1-3213e30d7be3 kind: Event lastTimestamp: "2026-05-25T13:16:30Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T13:16:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:16:30Z" name: isvc-xgboost-graph-predictor-669d8d6456-tzb6t.18b2d15ad9f04bcf namespace: kserve-ci-e2e-test resourceVersion: "15524" uid: 81c57b0f-c7f6-4b91-8a3d-7175c15ebf47 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-213.ec2.internal source: component: kubelet host: ip-10-0-130-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-tzb6t namespace: kserve-ci-e2e-test resourceVersion: "14108" uid: 105d3a7c-e7b0-46da-b3c1-3213e30d7be3 kind: Event lastTimestamp: "2026-05-25T13:16:30Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T13:16:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:16:30Z" name: isvc-xgboost-graph-predictor-669d8d6456-tzb6t.18b2d15ad9f0ee54 namespace: kserve-ci-e2e-test resourceVersion: "15527" uid: 28daa213-3fab-4cc0-8b0e-bf2cd441801b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-213.ec2.internal source: component: kubelet host: ip-10-0-130-213.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-tzb6t namespace: kserve-ci-e2e-test resourceVersion: "14108" uid: 105d3a7c-e7b0-46da-b3c1-3213e30d7be3 kind: Event lastTimestamp: "2026-05-25T13:16:30Z" message: 'Readiness probe failed: Get "https://10.133.0.17:8643/healthz": dial tcp 10.133.0.17:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-25T13:16:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:16:31Z" name: isvc-xgboost-graph-predictor-669d8d6456-tzb6t.18b2d15adfb44dc4 namespace: kserve-ci-e2e-test resourceVersion: "15565" uid: e2d21f49-dc7e-4d6f-8dd0-26c2ee75f9d0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-213.ec2.internal source: component: kubelet host: ip-10-0-130-213.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-669d8d6456 namespace: kserve-ci-e2e-test resourceVersion: "14094" uid: 28bb4323-a687-4cac-8868-93ff55d49f56 kind: Event lastTimestamp: "2026-05-25T13:14:31Z" message: 'Created pod: isvc-xgboost-graph-predictor-669d8d6456-tzb6t' metadata: creationTimestamp: "2026-05-25T13:14:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:14:31Z" name: isvc-xgboost-graph-predictor-669d8d6456.18b2d13efe579221 namespace: kserve-ci-e2e-test resourceVersion: "14107" uid: db99f145-c838-4d32-bcf5-5125856753f2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14092" uid: c68d916d-97d3-4aa8-b881-bd2fe81369bc kind: Event lastTimestamp: "2026-05-25T13:14:31Z" message: Scaled up replica set isvc-xgboost-graph-predictor-669d8d6456 from 0 to 1 metadata: creationTimestamp: "2026-05-25T13:14:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:14:31Z" name: isvc-xgboost-graph-predictor.18b2d13ef47f5fe6 namespace: kserve-ci-e2e-test resourceVersion: "14096" uid: 3f148dc1-6586-4ee6-b398-506fac310dca reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-25T13:14:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14093" uid: f6990665-270d-4a44-b3d8-e48054b5b127 kind: Event lastTimestamp: "2026-05-25T13:15:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T13:14:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:15:31Z" name: isvc-xgboost-graph-predictor.18b2d142730a2091 namespace: kserve-ci-e2e-test resourceVersion: "14739" uid: 02a729de-82c1-412b-a66d-6268ff5cbb38 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-25T13:14:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14093" uid: f6990665-270d-4a44-b3d8-e48054b5b127 kind: Event lastTimestamp: "2026-05-25T13:15:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T13:14:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:15:31Z" name: isvc-xgboost-graph-predictor.18b2d142737dc70e namespace: kserve-ci-e2e-test resourceVersion: "14740" uid: 407e3bc4-a774-4ae7-a8f4-7cb7a85e8d41 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T13:15:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14333" uid: f6990665-270d-4a44-b3d8-e48054b5b127 kind: Event lastTimestamp: "2026-05-25T13:16:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T13:15:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:16:16Z" name: isvc-xgboost-graph-predictor.18b2d1506e513159 namespace: kserve-ci-e2e-test resourceVersion: "15332" uid: af4833c5-8c03-46e8-8b30-2db7ab2f9be4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T13:15:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14333" uid: f6990665-270d-4a44-b3d8-e48054b5b127 kind: Event lastTimestamp: "2026-05-25T13:16:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T13:15:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:16:16Z" name: isvc-xgboost-graph-predictor.18b2d1506ebefad1 namespace: kserve-ci-e2e-test resourceVersion: "15333" uid: 3c0a5436-1ab7-4f61-8561-50f84cf97b1b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-05-25T13:14:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14072" uid: 681aa0d1-e453-4362-9df2-38e4207f4e4d kind: Event lastTimestamp: "2026-05-25T13:16:06Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T13:14:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:16:06Z" name: isvc-xgboost-graph.18b2d13ef46019fc namespace: kserve-ci-e2e-test resourceVersion: "15243" uid: eb30ec34-cb0d-4c4a-a07e-9f6b945a2d92 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15241" uid: 681aa0d1-e453-4362-9df2-38e4207f4e4d kind: Event lastTimestamp: "2026-05-25T13:16:06Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-05-25T13:16:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:16:06Z" name: isvc-xgboost-graph.18b2d15512b196d1 namespace: kserve-ci-e2e-test resourceVersion: "15242" uid: 4fe907de-b47b-45a7-8600-68d7323d2009 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T13:16:20.740929Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-866798f95b-9dhqz namespace: kserve-ci-e2e-test resourceVersion: "15386" uid: d8fe0506-8e9e-4e56-9b22-950b88f4e6f8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-866798f95b-9dhqz to ip-10-0-139-18.ec2.internal metadata: creationTimestamp: "2026-05-25T13:16:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T13:16:20Z" name: model-chainer-866798f95b-9dhqz.18b2d1587c813e28 namespace: kserve-ci-e2e-test resourceVersion: "15391" uid: 9dbb98a3-adcd-4725-a7c4-74f3f339d410 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c46b5f76-4gnkb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:20Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-866798f95b-9dhqz namespace: kserve-ci-e2e-test resourceVersion: "15388" uid: d8fe0506-8e9e-4e56-9b22-950b88f4e6f8 kind: Event lastTimestamp: "2026-05-25T13:16:20Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-serving-cert" not found' metadata: creationTimestamp: "2026-05-25T13:16:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:16:20Z" name: model-chainer-866798f95b-9dhqz.18b2d15885f1c82f namespace: kserve-ci-e2e-test resourceVersion: "15402" uid: 15977710-89e1-4caf-a52e-906fb2d762cb reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:21Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-866798f95b-9dhqz namespace: kserve-ci-e2e-test resourceVersion: "15394" uid: d8fe0506-8e9e-4e56-9b22-950b88f4e6f8 kind: Event lastTimestamp: "2026-05-25T13:16:21Z" message: Add eth0 [10.132.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T13:16:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T13:16:21Z" name: model-chainer-866798f95b-9dhqz.18b2d158ba322cd3 namespace: kserve-ci-e2e-test resourceVersion: "15411" uid: bd720dea-ad2d-4157-b989-c56aba22d6eb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-866798f95b-9dhqz namespace: kserve-ci-e2e-test resourceVersion: "15388" uid: d8fe0506-8e9e-4e56-9b22-950b88f4e6f8 kind: Event lastTimestamp: "2026-05-25T13:16:21Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:367463b1d3262f4416182663b2344c09f7ccedc3764563fd3d0aee32ec772aae" already present on machine metadata: creationTimestamp: "2026-05-25T13:16:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:16:21Z" name: model-chainer-866798f95b-9dhqz.18b2d158bb740525 namespace: kserve-ci-e2e-test resourceVersion: "15413" uid: 477d1846-dd2a-41fb-97f3-52fb53902cbc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-866798f95b-9dhqz namespace: kserve-ci-e2e-test resourceVersion: "15388" uid: d8fe0506-8e9e-4e56-9b22-950b88f4e6f8 kind: Event lastTimestamp: "2026-05-25T13:16:21Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-05-25T13:16:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:16:21Z" name: model-chainer-866798f95b-9dhqz.18b2d158bef5d833 namespace: kserve-ci-e2e-test resourceVersion: "15414" uid: dcab75eb-4f1e-4cb4-aec6-9ef6636eecda reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-866798f95b-9dhqz namespace: kserve-ci-e2e-test resourceVersion: "15388" uid: d8fe0506-8e9e-4e56-9b22-950b88f4e6f8 kind: Event lastTimestamp: "2026-05-25T13:16:21Z" message: Started container model-chainer metadata: creationTimestamp: "2026-05-25T13:16:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:16:21Z" name: model-chainer-866798f95b-9dhqz.18b2d158bfb10632 namespace: kserve-ci-e2e-test resourceVersion: "15416" uid: a7d82166-6c66-4ea7-bf2f-558145228658 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-866798f95b-9dhqz namespace: kserve-ci-e2e-test resourceVersion: "15388" uid: d8fe0506-8e9e-4e56-9b22-950b88f4e6f8 kind: Event lastTimestamp: "2026-05-25T13:16:30Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-05-25T13:16:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:16:30Z" name: model-chainer-866798f95b-9dhqz.18b2d15ad51308b8 namespace: kserve-ci-e2e-test resourceVersion: "15505" uid: ad731344-56b3-47c1-a337-8fee7a8f735d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T13:16:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-866798f95b-9dhqz namespace: kserve-ci-e2e-test resourceVersion: "15388" uid: d8fe0506-8e9e-4e56-9b22-950b88f4e6f8 kind: Event lastTimestamp: "2026-05-25T13:16:57Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-25T13:16:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:16:57Z" name: model-chainer-866798f95b-9dhqz.18b2d15b50270ebe namespace: kserve-ci-e2e-test resourceVersion: "15893" uid: aa6d4ed4-d8e1-49a9-8a5a-852eb42c5b63 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-866798f95b namespace: kserve-ci-e2e-test resourceVersion: "15383" uid: a2ac2443-d76a-493c-a096-4700f34cef49 kind: Event lastTimestamp: "2026-05-25T13:16:20Z" message: 'Created pod: model-chainer-866798f95b-9dhqz' metadata: creationTimestamp: "2026-05-25T13:16:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:16:20Z" name: model-chainer-866798f95b.18b2d1587bd32df9 namespace: kserve-ci-e2e-test resourceVersion: "15387" uid: a17227c5-7f87-46e0-abfb-b8b459121686 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15381" uid: fb64fa99-c989-48d5-8602-ff4afbf27c65 kind: Event lastTimestamp: "2026-05-25T13:16:20Z" message: Scaled up replica set model-chainer-866798f95b from 0 to 1 metadata: creationTimestamp: "2026-05-25T13:16:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:16:20Z" name: model-chainer.18b2d1587aac12cc namespace: kserve-ci-e2e-test resourceVersion: "15385" uid: 1484311e-7fc8-45ab-b6a2-89e366a09554 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:20Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15397" uid: b1b562f4-24ef-4d37-8d35-429c7b0c13f4 kind: Event lastTimestamp: "2026-05-25T13:16:20Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-05-25T13:16:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:16:20Z" name: model-chainer.18b2d15880d472e6 namespace: kserve-ci-e2e-test resourceVersion: "15399" uid: e1ff9cf3-0372-4f03-9f86-1db80c446dc7 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T13:17:31.006918Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-08c99-84bc9f87db-h8b2d namespace: kserve-ci-e2e-test resourceVersion: "16226" uid: c713f40d-97e5-4aef-9cde-11fcce6f1ec4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-08c99-84bc9f87db-h8b2d to ip-10-0-139-18.ec2.internal metadata: creationTimestamp: "2026-05-25T13:17:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T13:17:31Z" name: sequence-graph-08c99-84bc9f87db-h8b2d.18b2d168d8af265b namespace: kserve-ci-e2e-test resourceVersion: "16231" uid: 74f45fca-334b-44fe-8ef9-56534e37595c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c46b5f76-4gnkb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:17:31Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-08c99-84bc9f87db-h8b2d namespace: kserve-ci-e2e-test resourceVersion: "16235" uid: c713f40d-97e5-4aef-9cde-11fcce6f1ec4 kind: Event lastTimestamp: "2026-05-25T13:17:31Z" message: Add eth0 [10.132.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T13:17:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T13:17:31Z" name: sequence-graph-08c99-84bc9f87db-h8b2d.18b2d168f299c1a9 namespace: kserve-ci-e2e-test resourceVersion: "16254" uid: d59f2d64-e8a5-4e96-818f-792e8a269cbd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:17:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-08c99} kind: Pod name: sequence-graph-08c99-84bc9f87db-h8b2d namespace: kserve-ci-e2e-test resourceVersion: "16228" uid: c713f40d-97e5-4aef-9cde-11fcce6f1ec4 kind: Event lastTimestamp: "2026-05-25T13:17:31Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:367463b1d3262f4416182663b2344c09f7ccedc3764563fd3d0aee32ec772aae" already present on machine metadata: creationTimestamp: "2026-05-25T13:17:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:17:31Z" name: sequence-graph-08c99-84bc9f87db-h8b2d.18b2d168f406d067 namespace: kserve-ci-e2e-test resourceVersion: "16257" uid: fb3b3bab-5e3c-4b5e-a748-99f0e93cab7b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:17:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-08c99} kind: Pod name: sequence-graph-08c99-84bc9f87db-h8b2d namespace: kserve-ci-e2e-test resourceVersion: "16228" uid: c713f40d-97e5-4aef-9cde-11fcce6f1ec4 kind: Event lastTimestamp: "2026-05-25T13:17:31Z" message: 'Created container: sequence-graph-08c99' metadata: creationTimestamp: "2026-05-25T13:17:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:17:31Z" name: sequence-graph-08c99-84bc9f87db-h8b2d.18b2d168f70b29dc namespace: kserve-ci-e2e-test resourceVersion: "16258" uid: 68e880ab-8117-4f53-9365-7012a7d776cc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:17:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-08c99} kind: Pod name: sequence-graph-08c99-84bc9f87db-h8b2d namespace: kserve-ci-e2e-test resourceVersion: "16228" uid: c713f40d-97e5-4aef-9cde-11fcce6f1ec4 kind: Event lastTimestamp: "2026-05-25T13:17:31Z" message: Started container sequence-graph-08c99 metadata: creationTimestamp: "2026-05-25T13:17:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:17:31Z" name: sequence-graph-08c99-84bc9f87db-h8b2d.18b2d168f77e7561 namespace: kserve-ci-e2e-test resourceVersion: "16259" uid: 1c925f9e-9e32-4968-9a88-4b680ff0ff01 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:25:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-08c99} kind: Pod name: sequence-graph-08c99-84bc9f87db-h8b2d namespace: kserve-ci-e2e-test resourceVersion: "16228" uid: c713f40d-97e5-4aef-9cde-11fcce6f1ec4 kind: Event lastTimestamp: "2026-05-25T13:25:45Z" message: Stopping container sequence-graph-08c99 metadata: creationTimestamp: "2026-05-25T13:25:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:25:45Z" name: sequence-graph-08c99-84bc9f87db-h8b2d.18b2d1dc097ace34 namespace: kserve-ci-e2e-test resourceVersion: "19799" uid: 158446fa-74d4-487f-b334-7651fc68a724 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T13:25:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-08c99} kind: Pod name: sequence-graph-08c99-84bc9f87db-h8b2d namespace: kserve-ci-e2e-test resourceVersion: "16228" uid: c713f40d-97e5-4aef-9cde-11fcce6f1ec4 kind: Event lastTimestamp: "2026-05-25T13:26:13Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-25T13:25:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:26:13Z" name: sequence-graph-08c99-84bc9f87db-h8b2d.18b2d1dc9767aa5c namespace: kserve-ci-e2e-test resourceVersion: "20155" uid: 7b57d64d-f5b5-469d-a4d8-789b2c222e56 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:17:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-08c99-84bc9f87db namespace: kserve-ci-e2e-test resourceVersion: "16223" uid: 81f00bd5-fe90-4c4a-bdca-3a2f1f6e8bdd kind: Event lastTimestamp: "2026-05-25T13:17:30Z" message: 'Created pod: sequence-graph-08c99-84bc9f87db-h8b2d' metadata: creationTimestamp: "2026-05-25T13:17:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:17:30Z" name: sequence-graph-08c99-84bc9f87db.18b2d168d82a406e namespace: kserve-ci-e2e-test resourceVersion: "16227" uid: c06bb91d-84e2-48e9-9e90-12457c0ca0c2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:17:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-08c99 namespace: kserve-ci-e2e-test resourceVersion: "16221" uid: cc011526-6a47-431c-8931-a423e238e0ec kind: Event lastTimestamp: "2026-05-25T13:17:30Z" message: Scaled up replica set sequence-graph-08c99-84bc9f87db from 0 to 1 metadata: creationTimestamp: "2026-05-25T13:17:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:17:30Z" name: sequence-graph-08c99.18b2d168d7567580 namespace: kserve-ci-e2e-test resourceVersion: "16224" uid: 2bb34be4-4b6e-433a-a1eb-7fb099928bce reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:17:31Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-08c99 namespace: kserve-ci-e2e-test resourceVersion: "16243" uid: 4b3c0858-6a9f-462b-8ba2-7e37b1f9c79e kind: Event lastTimestamp: "2026-05-25T13:17:31Z" message: InferenceGraph [sequence-graph-08c99] is Ready metadata: creationTimestamp: "2026-05-25T13:17:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:17:31Z" name: sequence-graph-08c99.18b2d168de7488ed namespace: kserve-ci-e2e-test resourceVersion: "16247" uid: 811bc777-918e-4d71-8031-e20d2dbd36a3 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:17:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-08c99 namespace: kserve-ci-e2e-test resourceVersion: "16222" uid: e6d8c316-5a3b-4ed1-a413-51435d164dee kind: Event lastTimestamp: "2026-05-25T13:17:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T13:17:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:17:46Z" name: sequence-graph-08c99.18b2d16c56633643 namespace: kserve-ci-e2e-test resourceVersion: "16363" uid: 148a3354-908a-4f62-a2f1-2d95d9fa138b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:17:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-08c99 namespace: kserve-ci-e2e-test resourceVersion: "16222" uid: e6d8c316-5a3b-4ed1-a413-51435d164dee kind: Event lastTimestamp: "2026-05-25T13:17:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T13:17:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:17:46Z" name: sequence-graph-08c99.18b2d16c56ac5e70 namespace: kserve-ci-e2e-test resourceVersion: "16364" uid: 248dd704-6b26-4c81-9159-ac68ef869da4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:18:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-08c99 namespace: kserve-ci-e2e-test resourceVersion: "16362" uid: e6d8c316-5a3b-4ed1-a413-51435d164dee kind: Event lastTimestamp: "2026-05-25T13:18:01Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T13:18:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:18:01Z" name: sequence-graph-08c99.18b2d16fd5619851 namespace: kserve-ci-e2e-test resourceVersion: "16474" uid: ade21ebe-5d2a-43d8-88af-c5b5ca4f521f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:18:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-08c99 namespace: kserve-ci-e2e-test resourceVersion: "16362" uid: e6d8c316-5a3b-4ed1-a413-51435d164dee kind: Event lastTimestamp: "2026-05-25T13:18:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T13:18:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:18:01Z" name: sequence-graph-08c99.18b2d16fd5d0cbca namespace: kserve-ci-e2e-test resourceVersion: "16476" uid: d8f40988-eb1e-4cc6-b7da-e8224c2d1737 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-05-25T13:27:56.309138Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-72c4b-7c78954875-qkckb namespace: kserve-ci-e2e-test resourceVersion: "21463" uid: aa28c358-d76d-46fd-ae84-1f31f46e7fe7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-72c4b-7c78954875-qkckb to ip-10-0-139-18.ec2.internal metadata: creationTimestamp: "2026-05-25T13:27:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T13:27:56Z" name: sequence-graph-72c4b-7c78954875-qkckb.18b2d1fa6f99c710 namespace: kserve-ci-e2e-test resourceVersion: "21470" uid: 5d51fb94-be05-4335-a288-01f96d506b28 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c46b5f76-4gnkb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:27:56Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-72c4b-7c78954875-qkckb namespace: kserve-ci-e2e-test resourceVersion: "21474" uid: aa28c358-d76d-46fd-ae84-1f31f46e7fe7 kind: Event lastTimestamp: "2026-05-25T13:27:56Z" message: Add eth0 [10.132.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T13:27:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T13:27:56Z" name: sequence-graph-72c4b-7c78954875-qkckb.18b2d1fa896db605 namespace: kserve-ci-e2e-test resourceVersion: "21484" uid: 9dddd673-6918-4372-9543-35bc96625d58 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:27:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-72c4b} kind: Pod name: sequence-graph-72c4b-7c78954875-qkckb namespace: kserve-ci-e2e-test resourceVersion: "21468" uid: aa28c358-d76d-46fd-ae84-1f31f46e7fe7 kind: Event lastTimestamp: "2026-05-25T13:27:56Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:367463b1d3262f4416182663b2344c09f7ccedc3764563fd3d0aee32ec772aae" already present on machine metadata: creationTimestamp: "2026-05-25T13:27:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:27:56Z" name: sequence-graph-72c4b-7c78954875-qkckb.18b2d1fa8ae2baf4 namespace: kserve-ci-e2e-test resourceVersion: "21486" uid: f55807ec-5169-4724-b502-251b0b00e3c2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:27:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-72c4b} kind: Pod name: sequence-graph-72c4b-7c78954875-qkckb namespace: kserve-ci-e2e-test resourceVersion: "21468" uid: aa28c358-d76d-46fd-ae84-1f31f46e7fe7 kind: Event lastTimestamp: "2026-05-25T13:27:56Z" message: 'Created container: sequence-graph-72c4b' metadata: creationTimestamp: "2026-05-25T13:27:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:27:56Z" name: sequence-graph-72c4b-7c78954875-qkckb.18b2d1fa8dfae439 namespace: kserve-ci-e2e-test resourceVersion: "21489" uid: 4341c1ee-666e-425c-a33d-4e6ad62053ab reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:27:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-72c4b} kind: Pod name: sequence-graph-72c4b-7c78954875-qkckb namespace: kserve-ci-e2e-test resourceVersion: "21468" uid: aa28c358-d76d-46fd-ae84-1f31f46e7fe7 kind: Event lastTimestamp: "2026-05-25T13:27:56Z" message: Started container sequence-graph-72c4b metadata: creationTimestamp: "2026-05-25T13:27:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:27:56Z" name: sequence-graph-72c4b-7c78954875-qkckb.18b2d1fa8e72c44e namespace: kserve-ci-e2e-test resourceVersion: "21490" uid: f8d45f74-1298-414e-a67b-9f64b8efb88d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-72c4b} kind: Pod name: sequence-graph-72c4b-7c78954875-qkckb namespace: kserve-ci-e2e-test resourceVersion: "21468" uid: aa28c358-d76d-46fd-ae84-1f31f46e7fe7 kind: Event lastTimestamp: "2026-05-25T13:36:10Z" message: Stopping container sequence-graph-72c4b metadata: creationTimestamp: "2026-05-25T13:36:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:36:10Z" name: sequence-graph-72c4b-7c78954875-qkckb.18b2d26d9a02c7af namespace: kserve-ci-e2e-test resourceVersion: "25047" uid: 9f6d9f0e-6e1f-4581-b2ad-68fdc6d7d63f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T13:36:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-72c4b} kind: Pod name: sequence-graph-72c4b-7c78954875-qkckb namespace: kserve-ci-e2e-test resourceVersion: "21468" uid: aa28c358-d76d-46fd-ae84-1f31f46e7fe7 kind: Event lastTimestamp: "2026-05-25T13:36:38Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-25T13:36:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:36:38Z" name: sequence-graph-72c4b-7c78954875-qkckb.18b2d26e2693882e namespace: kserve-ci-e2e-test resourceVersion: "25416" uid: 32a66bfa-bb46-4bb5-992d-fa35430b9bcf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:27:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-72c4b-7c78954875 namespace: kserve-ci-e2e-test resourceVersion: "21459" uid: 3a48895f-331e-4beb-a6e2-0ea12f2bdd91 kind: Event lastTimestamp: "2026-05-25T13:27:56Z" message: 'Created pod: sequence-graph-72c4b-7c78954875-qkckb' metadata: creationTimestamp: "2026-05-25T13:27:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:27:56Z" name: sequence-graph-72c4b-7c78954875.18b2d1fa6ee005a7 namespace: kserve-ci-e2e-test resourceVersion: "21464" uid: 83c77507-e7fc-4006-9433-dd1ff4bf1e9c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:27:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-72c4b namespace: kserve-ci-e2e-test resourceVersion: "21457" uid: 5eac56a0-2903-4ccb-9790-53fd8299a3e6 kind: Event lastTimestamp: "2026-05-25T13:27:56Z" message: Scaled up replica set sequence-graph-72c4b-7c78954875 from 0 to 1 metadata: creationTimestamp: "2026-05-25T13:27:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:27:56Z" name: sequence-graph-72c4b.18b2d1fa6df8244d namespace: kserve-ci-e2e-test resourceVersion: "21461" uid: dd36ecf4-d530-4e70-943b-9298ef097125 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:27:56Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-72c4b namespace: kserve-ci-e2e-test resourceVersion: "21479" uid: 1c721524-7919-45bd-8a32-4d1eecc7587f kind: Event lastTimestamp: "2026-05-25T13:27:56Z" message: InferenceGraph [sequence-graph-72c4b] is Ready metadata: creationTimestamp: "2026-05-25T13:27:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:27:56Z" name: sequence-graph-72c4b.18b2d1fa74733f96 namespace: kserve-ci-e2e-test resourceVersion: "21481" uid: ce6270f6-adda-402a-b5b2-935ca2ef145c reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:28:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-72c4b namespace: kserve-ci-e2e-test resourceVersion: "21458" uid: 393f96bf-c811-4cbc-99e3-0dc7aab907f2 kind: Event lastTimestamp: "2026-05-25T13:28:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T13:28:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:28:11Z" name: sequence-graph-72c4b.18b2d1fdec3aa6e3 namespace: kserve-ci-e2e-test resourceVersion: "21595" uid: 8b4257fc-00f1-4528-96d0-20e737af9549 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:28:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-72c4b namespace: kserve-ci-e2e-test resourceVersion: "21458" uid: 393f96bf-c811-4cbc-99e3-0dc7aab907f2 kind: Event lastTimestamp: "2026-05-25T13:28:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T13:28:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:28:11Z" name: sequence-graph-72c4b.18b2d1fdeca150b8 namespace: kserve-ci-e2e-test resourceVersion: "21597" uid: 4e19f99d-89aa-44f0-b680-b63ce4c8f6b0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:28:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-72c4b namespace: kserve-ci-e2e-test resourceVersion: "21596" uid: 393f96bf-c811-4cbc-99e3-0dc7aab907f2 kind: Event lastTimestamp: "2026-05-25T13:28:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T13:28:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:28:26Z" name: sequence-graph-72c4b.18b2d2016b6842ba namespace: kserve-ci-e2e-test resourceVersion: "21712" uid: ef929123-4b32-4b6a-a316-24624a27b495 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:28:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-72c4b namespace: kserve-ci-e2e-test resourceVersion: "21596" uid: 393f96bf-c811-4cbc-99e3-0dc7aab907f2 kind: Event lastTimestamp: "2026-05-25T13:28:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T13:28:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:28:26Z" name: sequence-graph-72c4b.18b2d2016bd6dd82 namespace: kserve-ci-e2e-test resourceVersion: "21713" uid: 936b069b-b4a4-4db5-ac95-320f051c8f8d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-05-25T13:26:45.912665Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-9ee57-67b68b489d-bvwfw namespace: kserve-ci-e2e-test resourceVersion: "20641" uid: b7340801-9a36-4fe1-be78-44e58bb71d75 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-9ee57-67b68b489d-bvwfw to ip-10-0-139-18.ec2.internal metadata: creationTimestamp: "2026-05-25T13:26:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T13:26:45Z" name: sequence-graph-9ee57-67b68b489d-bvwfw.18b2d1ea0ba4cd1b namespace: kserve-ci-e2e-test resourceVersion: "20646" uid: bf650925-b273-4ba2-b19a-8cfa572911ec reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c46b5f76-4gnkb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:46Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-9ee57-67b68b489d-bvwfw namespace: kserve-ci-e2e-test resourceVersion: "20643" uid: b7340801-9a36-4fe1-be78-44e58bb71d75 kind: Event lastTimestamp: "2026-05-25T13:26:46Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-9ee57-serving-cert" not found' metadata: creationTimestamp: "2026-05-25T13:26:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:26:46Z" name: sequence-graph-9ee57-67b68b489d-bvwfw.18b2d1ea150d36cf namespace: kserve-ci-e2e-test resourceVersion: "20658" uid: 52626adc-73dd-48a3-a596-8306a1fdf17d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:46Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-9ee57-67b68b489d-bvwfw namespace: kserve-ci-e2e-test resourceVersion: "20652" uid: b7340801-9a36-4fe1-be78-44e58bb71d75 kind: Event lastTimestamp: "2026-05-25T13:26:46Z" message: Add eth0 [10.132.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T13:26:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T13:26:46Z" name: sequence-graph-9ee57-67b68b489d-bvwfw.18b2d1ea49559add namespace: kserve-ci-e2e-test resourceVersion: "20669" uid: 1aa95a21-e386-4b53-b545-a5497cff0fb8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-9ee57} kind: Pod name: sequence-graph-9ee57-67b68b489d-bvwfw namespace: kserve-ci-e2e-test resourceVersion: "20643" uid: b7340801-9a36-4fe1-be78-44e58bb71d75 kind: Event lastTimestamp: "2026-05-25T13:26:46Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:367463b1d3262f4416182663b2344c09f7ccedc3764563fd3d0aee32ec772aae" already present on machine metadata: creationTimestamp: "2026-05-25T13:26:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:26:46Z" name: sequence-graph-9ee57-67b68b489d-bvwfw.18b2d1ea4a955fe6 namespace: kserve-ci-e2e-test resourceVersion: "20671" uid: cc498b55-7180-425a-8625-434135c1c89e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-9ee57} kind: Pod name: sequence-graph-9ee57-67b68b489d-bvwfw namespace: kserve-ci-e2e-test resourceVersion: "20643" uid: b7340801-9a36-4fe1-be78-44e58bb71d75 kind: Event lastTimestamp: "2026-05-25T13:26:47Z" message: 'Created container: sequence-graph-9ee57' metadata: creationTimestamp: "2026-05-25T13:26:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:26:47Z" name: sequence-graph-9ee57-67b68b489d-bvwfw.18b2d1ea4da3fe3c namespace: kserve-ci-e2e-test resourceVersion: "20672" uid: 65d5013d-fd22-4bc5-91d8-5e09a2815c4e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-9ee57} kind: Pod name: sequence-graph-9ee57-67b68b489d-bvwfw namespace: kserve-ci-e2e-test resourceVersion: "20643" uid: b7340801-9a36-4fe1-be78-44e58bb71d75 kind: Event lastTimestamp: "2026-05-25T13:26:47Z" message: Started container sequence-graph-9ee57 metadata: creationTimestamp: "2026-05-25T13:26:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:26:47Z" name: sequence-graph-9ee57-67b68b489d-bvwfw.18b2d1ea4e1aaba6 namespace: kserve-ci-e2e-test resourceVersion: "20673" uid: 6fa7eee6-3c1e-4ee8-8c75-daf88996fd94 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-9ee57} kind: Pod name: sequence-graph-9ee57-67b68b489d-bvwfw namespace: kserve-ci-e2e-test resourceVersion: "20643" uid: b7340801-9a36-4fe1-be78-44e58bb71d75 kind: Event lastTimestamp: "2026-05-25T13:26:56Z" message: Stopping container sequence-graph-9ee57 metadata: creationTimestamp: "2026-05-25T13:26:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:26:56Z" name: sequence-graph-9ee57-67b68b489d-bvwfw.18b2d1ec657ce43b namespace: kserve-ci-e2e-test resourceVersion: "20763" uid: 12d920f7-48f4-4f91-88ec-a8ab5b6cbed7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T13:26:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-9ee57} kind: Pod name: sequence-graph-9ee57-67b68b489d-bvwfw namespace: kserve-ci-e2e-test resourceVersion: "20643" uid: b7340801-9a36-4fe1-be78-44e58bb71d75 kind: Event lastTimestamp: "2026-05-25T13:27:23Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-25T13:26:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:27:23Z" name: sequence-graph-9ee57-67b68b489d-bvwfw.18b2d1ecdee4773c namespace: kserve-ci-e2e-test resourceVersion: "21123" uid: 66327758-d00e-487b-bd4f-dc9f91934d6a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-9ee57-67b68b489d namespace: kserve-ci-e2e-test resourceVersion: "20638" uid: f32933f1-1f61-4823-af85-9c25a81d2b21 kind: Event lastTimestamp: "2026-05-25T13:26:45Z" message: 'Created pod: sequence-graph-9ee57-67b68b489d-bvwfw' metadata: creationTimestamp: "2026-05-25T13:26:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:26:45Z" name: sequence-graph-9ee57-67b68b489d.18b2d1ea0b40bf5c namespace: kserve-ci-e2e-test resourceVersion: "20642" uid: f6ad0fc0-14f3-494f-8d2d-4c52bef46b05 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-9ee57 namespace: kserve-ci-e2e-test resourceVersion: "20636" uid: 60a44283-8ac3-4617-a4e7-50c7ef982cb4 kind: Event lastTimestamp: "2026-05-25T13:26:45Z" message: Scaled up replica set sequence-graph-9ee57-67b68b489d from 0 to 1 metadata: creationTimestamp: "2026-05-25T13:26:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:26:45Z" name: sequence-graph-9ee57.18b2d1ea0a8d263d namespace: kserve-ci-e2e-test resourceVersion: "20639" uid: de320ca6-9a2c-44ae-b022-b874c4efadc6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:45Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-9ee57 namespace: kserve-ci-e2e-test resourceVersion: "20654" uid: eb0d642b-ffb4-4e26-ad9b-b5581710a964 kind: Event lastTimestamp: "2026-05-25T13:26:45Z" message: InferenceGraph [sequence-graph-9ee57] is Ready metadata: creationTimestamp: "2026-05-25T13:26:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:26:45Z" name: sequence-graph-9ee57.18b2d1ea0fe822ad namespace: kserve-ci-e2e-test resourceVersion: "20656" uid: bcc42716-3749-46c1-805a-5626675f873a reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T13:36:41.134507Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-35687-79cdffcd67-wpm88 namespace: kserve-ci-e2e-test resourceVersion: "25459" uid: 40d16003-4946-4138-bf3a-320fe9605fe6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-35687-79cdffcd67-wpm88 to ip-10-0-139-18.ec2.internal metadata: creationTimestamp: "2026-05-25T13:36:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T13:36:41Z" name: splitter-graph-35687-79cdffcd67-wpm88.18b2d274a1a1588b namespace: kserve-ci-e2e-test resourceVersion: "25464" uid: 112e2c5e-7ff2-4c53-bddf-abcc91e83180 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c46b5f76-4gnkb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:41Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-35687-79cdffcd67-wpm88 namespace: kserve-ci-e2e-test resourceVersion: "25470" uid: 40d16003-4946-4138-bf3a-320fe9605fe6 kind: Event lastTimestamp: "2026-05-25T13:36:41Z" message: Add eth0 [10.132.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T13:36:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T13:36:41Z" name: splitter-graph-35687-79cdffcd67-wpm88.18b2d274bbcff1db namespace: kserve-ci-e2e-test resourceVersion: "25486" uid: b988ed0c-4651-4c34-80ab-03939eada5ff reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-35687} kind: Pod name: splitter-graph-35687-79cdffcd67-wpm88 namespace: kserve-ci-e2e-test resourceVersion: "25461" uid: 40d16003-4946-4138-bf3a-320fe9605fe6 kind: Event lastTimestamp: "2026-05-25T13:36:41Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:367463b1d3262f4416182663b2344c09f7ccedc3764563fd3d0aee32ec772aae" already present on machine metadata: creationTimestamp: "2026-05-25T13:36:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:36:41Z" name: splitter-graph-35687-79cdffcd67-wpm88.18b2d274bd1a875c namespace: kserve-ci-e2e-test resourceVersion: "25488" uid: 3397559c-c2f5-4033-8c8b-e2592cf59cd2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-35687} kind: Pod name: splitter-graph-35687-79cdffcd67-wpm88 namespace: kserve-ci-e2e-test resourceVersion: "25461" uid: 40d16003-4946-4138-bf3a-320fe9605fe6 kind: Event lastTimestamp: "2026-05-25T13:36:41Z" message: 'Created container: splitter-graph-35687' metadata: creationTimestamp: "2026-05-25T13:36:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:36:41Z" name: splitter-graph-35687-79cdffcd67-wpm88.18b2d274c05421e5 namespace: kserve-ci-e2e-test resourceVersion: "25490" uid: 2e7b317e-5d37-4fce-bdc3-7c27019ba9c6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-35687} kind: Pod name: splitter-graph-35687-79cdffcd67-wpm88 namespace: kserve-ci-e2e-test resourceVersion: "25461" uid: 40d16003-4946-4138-bf3a-320fe9605fe6 kind: Event lastTimestamp: "2026-05-25T13:36:41Z" message: Started container splitter-graph-35687 metadata: creationTimestamp: "2026-05-25T13:36:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:36:41Z" name: splitter-graph-35687-79cdffcd67-wpm88.18b2d274c0d108ea namespace: kserve-ci-e2e-test resourceVersion: "25491" uid: a1afae75-f963-4e19-90c5-136544aa504c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-35687} kind: Pod name: splitter-graph-35687-79cdffcd67-wpm88 namespace: kserve-ci-e2e-test resourceVersion: "25461" uid: 40d16003-4946-4138-bf3a-320fe9605fe6 kind: Event lastTimestamp: "2026-05-25T13:36:51Z" message: Stopping container splitter-graph-35687 metadata: creationTimestamp: "2026-05-25T13:36:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:36:51Z" name: splitter-graph-35687-79cdffcd67-wpm88.18b2d276f9e7af33 namespace: kserve-ci-e2e-test resourceVersion: "25581" uid: 979e3aa5-f6d7-4105-9caf-4c8bf9d93e35 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T13:36:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-35687} kind: Pod name: splitter-graph-35687-79cdffcd67-wpm88 namespace: kserve-ci-e2e-test resourceVersion: "25461" uid: 40d16003-4946-4138-bf3a-320fe9605fe6 kind: Event lastTimestamp: "2026-05-25T13:37:18Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-25T13:36:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:37:18Z" name: splitter-graph-35687-79cdffcd67-wpm88.18b2d2776c8af5b1 namespace: kserve-ci-e2e-test resourceVersion: "26259" uid: ad91c0f3-123f-42fe-8482-73b8bc92951d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-35687-79cdffcd67 namespace: kserve-ci-e2e-test resourceVersion: "25454" uid: a540197b-9682-4f2d-b5ef-8a2d71b381ec kind: Event lastTimestamp: "2026-05-25T13:36:41Z" message: 'Created pod: splitter-graph-35687-79cdffcd67-wpm88' metadata: creationTimestamp: "2026-05-25T13:36:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:36:41Z" name: splitter-graph-35687-79cdffcd67.18b2d274a12e23c5 namespace: kserve-ci-e2e-test resourceVersion: "25460" uid: 68db8981-a01b-4f5d-872e-a2b2cf342909 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-35687 namespace: kserve-ci-e2e-test resourceVersion: "25451" uid: 08a66c8b-4fba-47f1-941c-398f1c1c93c6 kind: Event lastTimestamp: "2026-05-25T13:36:41Z" message: Scaled up replica set splitter-graph-35687-79cdffcd67 from 0 to 1 metadata: creationTimestamp: "2026-05-25T13:36:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:36:41Z" name: splitter-graph-35687.18b2d274a04d48f6 namespace: kserve-ci-e2e-test resourceVersion: "25456" uid: c9c1c11d-28bb-4461-94c1-c6fde1022eb8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:41Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-35687 namespace: kserve-ci-e2e-test resourceVersion: "25472" uid: 069c277b-7939-491d-8589-a8596afddc3d kind: Event lastTimestamp: "2026-05-25T13:36:41Z" message: InferenceGraph [splitter-graph-35687] is Ready metadata: creationTimestamp: "2026-05-25T13:36:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:36:41Z" name: splitter-graph-35687.18b2d274a4f92586 namespace: kserve-ci-e2e-test resourceVersion: "25474" uid: 5b759d35-e54f-4623-8f82-879c1e2a52fa reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T13:37:51.430879Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-766ab-98578b9d8-6bwk6 namespace: kserve-ci-e2e-test resourceVersion: "26551" uid: bec9eb96-a8e6-4b65-83df-5f49519d5235 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-766ab-98578b9d8-6bwk6 to ip-10-0-139-18.ec2.internal metadata: creationTimestamp: "2026-05-25T13:37:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T13:37:51Z" name: splitter-graph-766ab-98578b9d8-6bwk6.18b2d284ff9ee723 namespace: kserve-ci-e2e-test resourceVersion: "26557" uid: 2c0ad471-40cd-44a9-882c-88ef043607e8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c46b5f76-4gnkb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:37:51Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-766ab-98578b9d8-6bwk6 namespace: kserve-ci-e2e-test resourceVersion: "26562" uid: bec9eb96-a8e6-4b65-83df-5f49519d5235 kind: Event lastTimestamp: "2026-05-25T13:37:51Z" message: Add eth0 [10.132.0.47/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T13:37:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T13:37:51Z" name: splitter-graph-766ab-98578b9d8-6bwk6.18b2d2851915c8b0 namespace: kserve-ci-e2e-test resourceVersion: "26575" uid: fdad3fa0-30a0-489b-9acd-9b91fdf6f7b5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:37:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-766ab} kind: Pod name: splitter-graph-766ab-98578b9d8-6bwk6 namespace: kserve-ci-e2e-test resourceVersion: "26554" uid: bec9eb96-a8e6-4b65-83df-5f49519d5235 kind: Event lastTimestamp: "2026-05-25T13:37:51Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:367463b1d3262f4416182663b2344c09f7ccedc3764563fd3d0aee32ec772aae" already present on machine metadata: creationTimestamp: "2026-05-25T13:37:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:37:51Z" name: splitter-graph-766ab-98578b9d8-6bwk6.18b2d2851a620f26 namespace: kserve-ci-e2e-test resourceVersion: "26578" uid: ec77524a-702a-472d-9080-2cfd01c82877 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:37:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-766ab} kind: Pod name: splitter-graph-766ab-98578b9d8-6bwk6 namespace: kserve-ci-e2e-test resourceVersion: "26554" uid: bec9eb96-a8e6-4b65-83df-5f49519d5235 kind: Event lastTimestamp: "2026-05-25T13:37:51Z" message: 'Created container: splitter-graph-766ab' metadata: creationTimestamp: "2026-05-25T13:37:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:37:51Z" name: splitter-graph-766ab-98578b9d8-6bwk6.18b2d2851dd7f6ea namespace: kserve-ci-e2e-test resourceVersion: "26580" uid: 68749e88-f190-4be4-bdd4-f6e63bfed286 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:37:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-766ab} kind: Pod name: splitter-graph-766ab-98578b9d8-6bwk6 namespace: kserve-ci-e2e-test resourceVersion: "26554" uid: bec9eb96-a8e6-4b65-83df-5f49519d5235 kind: Event lastTimestamp: "2026-05-25T13:37:51Z" message: Started container splitter-graph-766ab metadata: creationTimestamp: "2026-05-25T13:37:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:37:51Z" name: splitter-graph-766ab-98578b9d8-6bwk6.18b2d2851e4f08ca namespace: kserve-ci-e2e-test resourceVersion: "26581" uid: 969304f9-b2fa-4457-a01a-cbca1b7f8003 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:46:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-766ab} kind: Pod name: splitter-graph-766ab-98578b9d8-6bwk6 namespace: kserve-ci-e2e-test resourceVersion: "26554" uid: bec9eb96-a8e6-4b65-83df-5f49519d5235 kind: Event lastTimestamp: "2026-05-25T13:46:06Z" message: Stopping container splitter-graph-766ab metadata: creationTimestamp: "2026-05-25T13:46:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:46:06Z" name: splitter-graph-766ab-98578b9d8-6bwk6.18b2d2f8311e450c namespace: kserve-ci-e2e-test resourceVersion: "30023" uid: 5ab78494-a056-471e-8cf1-907921246c82 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T13:46:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-766ab} kind: Pod name: splitter-graph-766ab-98578b9d8-6bwk6 namespace: kserve-ci-e2e-test resourceVersion: "26554" uid: bec9eb96-a8e6-4b65-83df-5f49519d5235 kind: Event lastTimestamp: "2026-05-25T13:46:33Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-25T13:46:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:46:33Z" name: splitter-graph-766ab-98578b9d8-6bwk6.18b2d2f8b474e73a namespace: kserve-ci-e2e-test resourceVersion: "30253" uid: 8d6c8742-051b-43e4-acb2-6ef2439c69f1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:37:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-766ab-98578b9d8 namespace: kserve-ci-e2e-test resourceVersion: "26548" uid: 2cc37b97-ed70-4fb8-a117-f7a2fb0294ae kind: Event lastTimestamp: "2026-05-25T13:37:51Z" message: 'Created pod: splitter-graph-766ab-98578b9d8-6bwk6' metadata: creationTimestamp: "2026-05-25T13:37:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:37:51Z" name: splitter-graph-766ab-98578b9d8.18b2d284ff3734f1 namespace: kserve-ci-e2e-test resourceVersion: "26553" uid: bec38bcb-8d46-423a-a7fb-bf1f49bd81e6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:37:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-766ab namespace: kserve-ci-e2e-test resourceVersion: "26546" uid: 6ff58e49-2978-4fd8-8c4f-be54039d566a kind: Event lastTimestamp: "2026-05-25T13:37:51Z" message: Scaled up replica set splitter-graph-766ab-98578b9d8 from 0 to 1 metadata: creationTimestamp: "2026-05-25T13:37:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:37:51Z" name: splitter-graph-766ab.18b2d284fe46e143 namespace: kserve-ci-e2e-test resourceVersion: "26549" uid: e576d116-ec9e-4d8e-bd46-672a825320f2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:37:51Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-766ab namespace: kserve-ci-e2e-test resourceVersion: "26567" uid: 2fc61292-d386-403c-a4bf-b83c437bd932 kind: Event lastTimestamp: "2026-05-25T13:37:51Z" message: InferenceGraph [splitter-graph-766ab] is Ready metadata: creationTimestamp: "2026-05-25T13:37:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:37:51Z" name: splitter-graph-766ab.18b2d285037351b5 namespace: kserve-ci-e2e-test resourceVersion: "26570" uid: be280e62-ec53-4e33-b922-563c20b6d020 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:38:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-766ab namespace: kserve-ci-e2e-test resourceVersion: "26547" uid: b4847704-986a-40dd-a295-7f266d3d8fc9 kind: Event lastTimestamp: "2026-05-25T13:38:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T13:38:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:38:06Z" name: splitter-graph-766ab.18b2d2887ca3fb01 namespace: kserve-ci-e2e-test resourceVersion: "26690" uid: 79b89bb2-b7ec-4af6-b7f9-155b7595152b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:38:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-766ab namespace: kserve-ci-e2e-test resourceVersion: "26547" uid: b4847704-986a-40dd-a295-7f266d3d8fc9 kind: Event lastTimestamp: "2026-05-25T13:38:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T13:38:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:38:06Z" name: splitter-graph-766ab.18b2d2887d089fbe namespace: kserve-ci-e2e-test resourceVersion: "26692" uid: bdd03c70-e641-4d40-9da1-513cafa5d1e9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:38:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-766ab namespace: kserve-ci-e2e-test resourceVersion: "26691" uid: b4847704-986a-40dd-a295-7f266d3d8fc9 kind: Event lastTimestamp: "2026-05-25T13:38:21Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T13:38:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:38:21Z" name: splitter-graph-766ab.18b2d28bfba50d11 namespace: kserve-ci-e2e-test resourceVersion: "26809" uid: 2add22d3-c0d2-474f-902f-20fd06488f70 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:38:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-766ab namespace: kserve-ci-e2e-test resourceVersion: "26691" uid: b4847704-986a-40dd-a295-7f266d3d8fc9 kind: Event lastTimestamp: "2026-05-25T13:38:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T13:38:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:38:21Z" name: splitter-graph-766ab.18b2d28bfbed6071 namespace: kserve-ci-e2e-test resourceVersion: "26811" uid: 52b07932-2303-4a13-b8ce-730dbb967c52 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-05-25T13:16:31.013580Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-08c99-predictor-cc5d7f476-czxk9 namespace: kserve-ci-e2e-test resourceVersion: "15559" uid: 11a87d3c-4909-445a-ab48-fb162c262188 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-08c99-predictor-cc5d7f476-czxk9 to ip-10-0-135-103.ec2.internal metadata: creationTimestamp: "2026-05-25T13:16:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T13:16:31Z" name: success-200-isvc-08c99-predictor-cc5d7f476-czxk9.18b2d15ae0cd6899 namespace: kserve-ci-e2e-test resourceVersion: "15574" uid: 1f584071-85c0-4a3b-950b-dca4299e6701 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c46b5f76-4gnkb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:31Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-08c99-predictor-cc5d7f476-czxk9 namespace: kserve-ci-e2e-test resourceVersion: "15589" uid: 11a87d3c-4909-445a-ab48-fb162c262188 kind: Event lastTimestamp: "2026-05-25T13:16:31Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T13:16:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T13:16:31Z" name: success-200-isvc-08c99-predictor-cc5d7f476-czxk9.18b2d15afad99911 namespace: kserve-ci-e2e-test resourceVersion: "15631" uid: c3e11de3-da4f-4290-aec7-7166ed3e41a1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-08c99-predictor-cc5d7f476-czxk9 namespace: kserve-ci-e2e-test resourceVersion: "15567" uid: 11a87d3c-4909-445a-ab48-fb162c262188 kind: Event lastTimestamp: "2026-05-25T13:16:31Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-25T13:16:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:16:31Z" name: success-200-isvc-08c99-predictor-cc5d7f476-czxk9.18b2d15afc529b50 namespace: kserve-ci-e2e-test resourceVersion: "15635" uid: b429995b-4e73-476f-ba51-e311dc333093 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-08c99-predictor-cc5d7f476-czxk9 namespace: kserve-ci-e2e-test resourceVersion: "15567" uid: 11a87d3c-4909-445a-ab48-fb162c262188 kind: Event lastTimestamp: "2026-05-25T13:16:31Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T13:16:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:16:31Z" name: success-200-isvc-08c99-predictor-cc5d7f476-czxk9.18b2d15aff44a8e2 namespace: kserve-ci-e2e-test resourceVersion: "15636" uid: f18f86d9-2fee-4ba8-b93c-81cfc932c783 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-08c99-predictor-cc5d7f476-czxk9 namespace: kserve-ci-e2e-test resourceVersion: "15567" uid: 11a87d3c-4909-445a-ab48-fb162c262188 kind: Event lastTimestamp: "2026-05-25T13:16:31Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T13:16:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:16:31Z" name: success-200-isvc-08c99-predictor-cc5d7f476-czxk9.18b2d15affb8228a namespace: kserve-ci-e2e-test resourceVersion: "15637" uid: 93d43695-9f3a-4a69-a07b-3e701366dcfc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-08c99-predictor-cc5d7f476-czxk9 namespace: kserve-ci-e2e-test resourceVersion: "15567" uid: 11a87d3c-4909-445a-ab48-fb162c262188 kind: Event lastTimestamp: "2026-05-25T13:16:31Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-25T13:16:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:16:31Z" name: success-200-isvc-08c99-predictor-cc5d7f476-czxk9.18b2d15affc404ae namespace: kserve-ci-e2e-test resourceVersion: "15638" uid: c59d4b34-4add-4b65-a4fe-5ba14cc93839 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-08c99-predictor-cc5d7f476-czxk9 namespace: kserve-ci-e2e-test resourceVersion: "15567" uid: 11a87d3c-4909-445a-ab48-fb162c262188 kind: Event lastTimestamp: "2026-05-25T13:16:31Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T13:16:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:16:31Z" name: success-200-isvc-08c99-predictor-cc5d7f476-czxk9.18b2d15b03575c87 namespace: kserve-ci-e2e-test resourceVersion: "15639" uid: 0fc3bcf0-0a2b-476e-a288-4ce6c23cbf4c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-08c99-predictor-cc5d7f476-czxk9 namespace: kserve-ci-e2e-test resourceVersion: "15567" uid: 11a87d3c-4909-445a-ab48-fb162c262188 kind: Event lastTimestamp: "2026-05-25T13:16:31Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T13:16:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:16:31Z" name: success-200-isvc-08c99-predictor-cc5d7f476-czxk9.18b2d15b03cfbad9 namespace: kserve-ci-e2e-test resourceVersion: "15640" uid: b4e8cdfd-5324-4d3d-beea-163c84ab1395 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-05-25T13:16:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-08c99-predictor-cc5d7f476-czxk9 namespace: kserve-ci-e2e-test resourceVersion: "15567" uid: 11a87d3c-4909-445a-ab48-fb162c262188 kind: Event lastTimestamp: "2026-05-25T13:25:48Z" message: 'Readiness probe failed: dial tcp 10.134.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T13:16:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:25:48Z" name: success-200-isvc-08c99-predictor-cc5d7f476-czxk9.18b2d15b4a9353a7 namespace: kserve-ci-e2e-test resourceVersion: "19947" uid: 0b76a3f9-e5cc-4622-bb2c-970fcc26b97c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:25:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-08c99-predictor-cc5d7f476-czxk9 namespace: kserve-ci-e2e-test resourceVersion: "15567" uid: 11a87d3c-4909-445a-ab48-fb162c262188 kind: Event lastTimestamp: "2026-05-25T13:25:45Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T13:25:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:25:45Z" name: success-200-isvc-08c99-predictor-cc5d7f476-czxk9.18b2d1dc0eb765a5 namespace: kserve-ci-e2e-test resourceVersion: "19820" uid: d925d3ac-4ea6-41d7-95e8-db77b987e63c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:25:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-08c99-predictor-cc5d7f476-czxk9 namespace: kserve-ci-e2e-test resourceVersion: "15567" uid: 11a87d3c-4909-445a-ab48-fb162c262188 kind: Event lastTimestamp: "2026-05-25T13:25:45Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T13:25:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:25:45Z" name: success-200-isvc-08c99-predictor-cc5d7f476-czxk9.18b2d1dc0eb81af6 namespace: kserve-ci-e2e-test resourceVersion: "19823" uid: c8bfa4dd-f639-49ad-93dd-a19e538345dd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:25:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-08c99-predictor-cc5d7f476-czxk9 namespace: kserve-ci-e2e-test resourceVersion: "15567" uid: 11a87d3c-4909-445a-ab48-fb162c262188 kind: Event lastTimestamp: "2026-05-25T13:25:48Z" message: 'Readiness probe failed: Get "https://10.134.0.36:8643/healthz": dial tcp 10.134.0.36:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-25T13:25:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:25:48Z" name: success-200-isvc-08c99-predictor-cc5d7f476-czxk9.18b2d1dcbef7f004 namespace: kserve-ci-e2e-test resourceVersion: "19946" uid: 00f322ef-39e1-44e5-a82a-01ce57c28fbc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-08c99-predictor-cc5d7f476 namespace: kserve-ci-e2e-test resourceVersion: "15548" uid: 49ad8db2-74c0-4de0-b758-0f985dbbc38b kind: Event lastTimestamp: "2026-05-25T13:16:30Z" message: 'Created pod: success-200-isvc-08c99-predictor-cc5d7f476-czxk9' metadata: creationTimestamp: "2026-05-25T13:16:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:16:31Z" name: success-200-isvc-08c99-predictor-cc5d7f476.18b2d15adfe4eceb namespace: kserve-ci-e2e-test resourceVersion: "15566" uid: 27582289-e0f0-49d7-9c4f-8dba5491c6f3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-08c99-predictor namespace: kserve-ci-e2e-test resourceVersion: "15541" uid: 6535b99f-c6eb-4056-b88a-113755c2a242 kind: Event lastTimestamp: "2026-05-25T13:16:30Z" message: Scaled up replica set success-200-isvc-08c99-predictor-cc5d7f476 from 0 to 1 metadata: creationTimestamp: "2026-05-25T13:16:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:16:30Z" name: success-200-isvc-08c99-predictor.18b2d15adcf1d3ef namespace: kserve-ci-e2e-test resourceVersion: "15551" uid: d2aa3253-6e35-4666-9b2c-24aad5024cb8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-08c99-predictor namespace: kserve-ci-e2e-test resourceVersion: "15547" uid: 1e9bb547-0d72-42c0-bfdb-020cfc4eba62 kind: Event lastTimestamp: "2026-05-25T13:16:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T13:16:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:16:45Z" name: success-200-isvc-08c99-predictor.18b2d15e5d12d396 namespace: kserve-ci-e2e-test resourceVersion: "15785" uid: 65369731-42bc-4776-a3cd-c55667f3077d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-08c99-predictor namespace: kserve-ci-e2e-test resourceVersion: "15547" uid: 1e9bb547-0d72-42c0-bfdb-020cfc4eba62 kind: Event lastTimestamp: "2026-05-25T13:16:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T13:16:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:16:45Z" name: success-200-isvc-08c99-predictor.18b2d15e5d7bfb34 namespace: kserve-ci-e2e-test resourceVersion: "15786" uid: 8f1b364b-5775-48a4-8e26-298ab6bfcdd8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T13:17:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-08c99-predictor namespace: kserve-ci-e2e-test resourceVersion: "15784" uid: 1e9bb547-0d72-42c0-bfdb-020cfc4eba62 kind: Event lastTimestamp: "2026-05-25T13:17:31Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T13:17:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:17:31Z" name: success-200-isvc-08c99-predictor.18b2d161dc23197c namespace: kserve-ci-e2e-test resourceVersion: "16236" uid: 1afb2b52-8947-4b16-9a10-2468d21b1050 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T13:17:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-08c99-predictor namespace: kserve-ci-e2e-test resourceVersion: "15784" uid: 1e9bb547-0d72-42c0-bfdb-020cfc4eba62 kind: Event lastTimestamp: "2026-05-25T13:17:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T13:17:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:17:31Z" name: success-200-isvc-08c99-predictor.18b2d161dc95c15a namespace: kserve-ci-e2e-test resourceVersion: "16237" uid: 6116362f-48bd-4520-966e-503303ef173d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T13:16:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-08c99 namespace: kserve-ci-e2e-test resourceVersion: "15516" uid: dd113761-a5db-4b0c-a6f4-8ceb4048a48c kind: Event lastTimestamp: "2026-05-25T13:17:18Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T13:16:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:17:18Z" name: success-200-isvc-08c99.18b2d15adcc17c69 namespace: kserve-ci-e2e-test resourceVersion: "16118" uid: ae55197f-ae8a-45c3-95d3-ebcb09fce90a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:17:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-08c99 namespace: kserve-ci-e2e-test resourceVersion: "16116" uid: dd113761-a5db-4b0c-a6f4-8ceb4048a48c kind: Event lastTimestamp: "2026-05-25T13:17:18Z" message: InferenceService [success-200-isvc-08c99] is Ready metadata: creationTimestamp: "2026-05-25T13:17:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:17:18Z" name: success-200-isvc-08c99.18b2d166054ab83c namespace: kserve-ci-e2e-test resourceVersion: "16117" uid: 69c351c5-052b-4e3a-b2e9-0fa97c1250bc reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T13:36:11.106818Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-12313-predictor-857dcf8cc4-46rl6 namespace: kserve-ci-e2e-test resourceVersion: "25087" uid: 51711b09-0032-45d2-99eb-00e0eff727e0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-12313-predictor-857dcf8cc4-46rl6 to ip-10-0-135-103.ec2.internal metadata: creationTimestamp: "2026-05-25T13:36:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T13:36:11Z" name: success-200-isvc-12313-predictor-857dcf8cc4-46rl6.18b2d26da3d7374d namespace: kserve-ci-e2e-test resourceVersion: "25094" uid: b03b25f5-07fe-4f1f-9013-c7ffeb787c98 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c46b5f76-4gnkb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:11Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-12313-predictor-857dcf8cc4-46rl6 namespace: kserve-ci-e2e-test resourceVersion: "25106" uid: 51711b09-0032-45d2-99eb-00e0eff727e0 kind: Event lastTimestamp: "2026-05-25T13:36:11Z" message: Add eth0 [10.134.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T13:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T13:36:11Z" name: success-200-isvc-12313-predictor-857dcf8cc4-46rl6.18b2d26dbda8723f namespace: kserve-ci-e2e-test resourceVersion: "25148" uid: 2564aaaf-b3fe-42b0-96d6-f45488b9af81 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-12313-predictor-857dcf8cc4-46rl6 namespace: kserve-ci-e2e-test resourceVersion: "25089" uid: 51711b09-0032-45d2-99eb-00e0eff727e0 kind: Event lastTimestamp: "2026-05-25T13:36:11Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-25T13:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:36:11Z" name: success-200-isvc-12313-predictor-857dcf8cc4-46rl6.18b2d26dbf2a2b05 namespace: kserve-ci-e2e-test resourceVersion: "25150" uid: f975ae39-6a57-42df-a5ec-50636007b7ef reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-12313-predictor-857dcf8cc4-46rl6 namespace: kserve-ci-e2e-test resourceVersion: "25089" uid: 51711b09-0032-45d2-99eb-00e0eff727e0 kind: Event lastTimestamp: "2026-05-25T13:36:11Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T13:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:36:11Z" name: success-200-isvc-12313-predictor-857dcf8cc4-46rl6.18b2d26dc2a4503e namespace: kserve-ci-e2e-test resourceVersion: "25151" uid: f17bd13b-61f3-4181-9f0c-7706a4274785 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-12313-predictor-857dcf8cc4-46rl6 namespace: kserve-ci-e2e-test resourceVersion: "25089" uid: 51711b09-0032-45d2-99eb-00e0eff727e0 kind: Event lastTimestamp: "2026-05-25T13:36:11Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T13:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:36:11Z" name: success-200-isvc-12313-predictor-857dcf8cc4-46rl6.18b2d26dc321a4c1 namespace: kserve-ci-e2e-test resourceVersion: "25152" uid: 9c8018bd-99cc-48d0-b0a1-3a45d3f232e6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-12313-predictor-857dcf8cc4-46rl6 namespace: kserve-ci-e2e-test resourceVersion: "25089" uid: 51711b09-0032-45d2-99eb-00e0eff727e0 kind: Event lastTimestamp: "2026-05-25T13:36:11Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-25T13:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:36:11Z" name: success-200-isvc-12313-predictor-857dcf8cc4-46rl6.18b2d26dc32ddd21 namespace: kserve-ci-e2e-test resourceVersion: "25153" uid: 907979e9-4f56-4c52-bba1-d3facce3326c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-12313-predictor-857dcf8cc4-46rl6 namespace: kserve-ci-e2e-test resourceVersion: "25089" uid: 51711b09-0032-45d2-99eb-00e0eff727e0 kind: Event lastTimestamp: "2026-05-25T13:36:11Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T13:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:36:11Z" name: success-200-isvc-12313-predictor-857dcf8cc4-46rl6.18b2d26dc5de756f namespace: kserve-ci-e2e-test resourceVersion: "25155" uid: 943f023b-2bb6-44f5-bf91-ec78e5285ad8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-12313-predictor-857dcf8cc4-46rl6 namespace: kserve-ci-e2e-test resourceVersion: "25089" uid: 51711b09-0032-45d2-99eb-00e0eff727e0 kind: Event lastTimestamp: "2026-05-25T13:36:11Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T13:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:36:11Z" name: success-200-isvc-12313-predictor-857dcf8cc4-46rl6.18b2d26dc6da96ed namespace: kserve-ci-e2e-test resourceVersion: "25156" uid: 56876844-a98e-41ab-8d19-fb2f6a74e0b8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T13:36:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-12313-predictor-857dcf8cc4-46rl6 namespace: kserve-ci-e2e-test resourceVersion: "25089" uid: 51711b09-0032-45d2-99eb-00e0eff727e0 kind: Event lastTimestamp: "2026-05-25T13:36:48Z" message: 'Readiness probe failed: dial tcp 10.134.0.42:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T13:36:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:36:48Z" name: success-200-isvc-12313-predictor-857dcf8cc4-46rl6.18b2d26e0aa7c6a1 namespace: kserve-ci-e2e-test resourceVersion: "25545" uid: 03e710e1-9c03-4d3e-a68e-c76ad8e8da25 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:53:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-12313-predictor-857dcf8cc4-46rl6 namespace: kserve-ci-e2e-test resourceVersion: "25089" uid: 51711b09-0032-45d2-99eb-00e0eff727e0 kind: Event lastTimestamp: "2026-05-25T13:53:30Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T13:53:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:53:30Z" name: success-200-isvc-12313-predictor-857dcf8cc4-46rl6.18b2d35fb0471f78 namespace: kserve-ci-e2e-test resourceVersion: "33117" uid: 41c67703-b2fd-468b-a8bf-41a1c797cb57 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:53:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-12313-predictor-857dcf8cc4-46rl6 namespace: kserve-ci-e2e-test resourceVersion: "25089" uid: 51711b09-0032-45d2-99eb-00e0eff727e0 kind: Event lastTimestamp: "2026-05-25T13:53:30Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T13:53:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:53:30Z" name: success-200-isvc-12313-predictor-857dcf8cc4-46rl6.18b2d35fb047ab8b namespace: kserve-ci-e2e-test resourceVersion: "33118" uid: 8d38a185-056c-4e90-904f-e834c1f3ae88 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-12313-predictor-857dcf8cc4 namespace: kserve-ci-e2e-test resourceVersion: "25071" uid: 10b25e01-23ce-4c62-82d6-20603235d472 kind: Event lastTimestamp: "2026-05-25T13:36:11Z" message: 'Created pod: success-200-isvc-12313-predictor-857dcf8cc4-46rl6' metadata: creationTimestamp: "2026-05-25T13:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:36:11Z" name: success-200-isvc-12313-predictor-857dcf8cc4.18b2d26da34500cc namespace: kserve-ci-e2e-test resourceVersion: "25091" uid: 3d601459-5c74-455c-ace7-1768ec6a254c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-12313-predictor namespace: kserve-ci-e2e-test resourceVersion: "25065" uid: 3d285a91-59e8-4680-b029-ffb077fb08f3 kind: Event lastTimestamp: "2026-05-25T13:36:11Z" message: Scaled up replica set success-200-isvc-12313-predictor-857dcf8cc4 from 0 to 1 metadata: creationTimestamp: "2026-05-25T13:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:36:11Z" name: success-200-isvc-12313-predictor.18b2d26da0681c18 namespace: kserve-ci-e2e-test resourceVersion: "25076" uid: 853550d8-f24e-44c4-b779-e5439000bc0e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T13:36:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-12313-predictor namespace: kserve-ci-e2e-test resourceVersion: "25070" uid: ff258761-d396-4d6e-a160-b9989eedee85 kind: Event lastTimestamp: "2026-05-25T13:36:41Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T13:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:36:41Z" name: success-200-isvc-12313-predictor.18b2d2711f16e03e namespace: kserve-ci-e2e-test resourceVersion: "25447" uid: 2e3db686-4a9e-420e-876c-883dbc672af2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T13:36:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-12313-predictor namespace: kserve-ci-e2e-test resourceVersion: "25070" uid: ff258761-d396-4d6e-a160-b9989eedee85 kind: Event lastTimestamp: "2026-05-25T13:36:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T13:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:36:41Z" name: success-200-isvc-12313-predictor.18b2d2711f813ea1 namespace: kserve-ci-e2e-test resourceVersion: "25452" uid: 07fbc6f0-c295-42e8-a526-01b28851d0c0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T13:36:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-12313-predictor namespace: kserve-ci-e2e-test resourceVersion: "25301" uid: ff258761-d396-4d6e-a160-b9989eedee85 kind: Event lastTimestamp: "2026-05-25T13:37:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T13:36:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:37:11Z" name: success-200-isvc-12313-predictor.18b2d2781dc6503e namespace: kserve-ci-e2e-test resourceVersion: "26182" uid: a7a063eb-409d-4a25-9cad-ed2fb97e6f3b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T13:36:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-12313-predictor namespace: kserve-ci-e2e-test resourceVersion: "25301" uid: ff258761-d396-4d6e-a160-b9989eedee85 kind: Event lastTimestamp: "2026-05-25T13:37:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T13:36:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:37:11Z" name: success-200-isvc-12313-predictor.18b2d2781e492db2 namespace: kserve-ci-e2e-test resourceVersion: "26186" uid: b2bc7ea5-bd05-4b2b-b854-6a834651e850 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T13:36:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-12313 namespace: kserve-ci-e2e-test resourceVersion: "25049" uid: 2f550f16-9526-468f-8b5e-03574fc6b850 kind: Event lastTimestamp: "2026-05-25T13:36:58Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T13:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:36:58Z" name: success-200-isvc-12313.18b2d26da030a244 namespace: kserve-ci-e2e-test resourceVersion: "25832" uid: f6c3a986-6c72-4376-96b3-d5f93efd4679 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-12313 namespace: kserve-ci-e2e-test resourceVersion: "25049" uid: 2f550f16-9526-468f-8b5e-03574fc6b850 kind: Event lastTimestamp: "2026-05-25T13:36:11Z" message: 'Failed to update status for InferenceService "success-200-isvc-12313": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-12313": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T13:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:36:11Z" name: success-200-isvc-12313.18b2d26da0886441 namespace: kserve-ci-e2e-test resourceVersion: "25079" uid: 1d2ab636-4132-4c5b-bf94-6088784872d1 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-12313 namespace: kserve-ci-e2e-test resourceVersion: "25049" uid: 2f550f16-9526-468f-8b5e-03574fc6b850 kind: Event lastTimestamp: "2026-05-25T13:36:11Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-12313": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T13:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:36:11Z" name: success-200-isvc-12313.18b2d26da088c99b namespace: kserve-ci-e2e-test resourceVersion: "25082" uid: 75466bb3-dfa3-4e58-8eff-60c36d2fba5b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-12313 namespace: kserve-ci-e2e-test resourceVersion: "25830" uid: 2f550f16-9526-468f-8b5e-03574fc6b850 kind: Event lastTimestamp: "2026-05-25T13:36:58Z" message: InferenceService [success-200-isvc-12313] is Ready metadata: creationTimestamp: "2026-05-25T13:36:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:36:58Z" name: success-200-isvc-12313.18b2d278c542546d namespace: kserve-ci-e2e-test resourceVersion: "25831" uid: bb1edbab-27e3-4c9d-96bc-53dca7c3f50c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T13:16:00.996259Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-339a4-predictor-58f8586498-9r8dr namespace: kserve-ci-e2e-test resourceVersion: "15113" uid: a3abc8ef-5a38-4ebc-bdbb-1c16bf6e82a0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-339a4-predictor-58f8586498-9r8dr to ip-10-0-135-103.ec2.internal metadata: creationTimestamp: "2026-05-25T13:16:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T13:16:00Z" name: success-200-isvc-339a4-predictor-58f8586498-9r8dr.18b2d153e3a17558 namespace: kserve-ci-e2e-test resourceVersion: "15118" uid: 05906e0a-5cd4-4e8f-a23e-ffb824e63b1b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c46b5f76-4gnkb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:01Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-339a4-predictor-58f8586498-9r8dr namespace: kserve-ci-e2e-test resourceVersion: "15123" uid: a3abc8ef-5a38-4ebc-bdbb-1c16bf6e82a0 kind: Event lastTimestamp: "2026-05-25T13:16:01Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T13:16:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T13:16:01Z" name: success-200-isvc-339a4-predictor-58f8586498-9r8dr.18b2d153fd40d2d4 namespace: kserve-ci-e2e-test resourceVersion: "15162" uid: e91754bc-b06f-44b0-a587-b0ff639f8910 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-339a4-predictor-58f8586498-9r8dr namespace: kserve-ci-e2e-test resourceVersion: "15115" uid: a3abc8ef-5a38-4ebc-bdbb-1c16bf6e82a0 kind: Event lastTimestamp: "2026-05-25T13:16:01Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-25T13:16:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:16:01Z" name: success-200-isvc-339a4-predictor-58f8586498-9r8dr.18b2d153feaddd37 namespace: kserve-ci-e2e-test resourceVersion: "15164" uid: 7835f0d9-9095-4c87-9059-01464888d089 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-339a4-predictor-58f8586498-9r8dr namespace: kserve-ci-e2e-test resourceVersion: "15115" uid: a3abc8ef-5a38-4ebc-bdbb-1c16bf6e82a0 kind: Event lastTimestamp: "2026-05-25T13:16:01Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T13:16:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:16:01Z" name: success-200-isvc-339a4-predictor-58f8586498-9r8dr.18b2d15401663769 namespace: kserve-ci-e2e-test resourceVersion: "15166" uid: c5bdd9b8-aa84-4bdd-802f-6d01b00997e7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-339a4-predictor-58f8586498-9r8dr namespace: kserve-ci-e2e-test resourceVersion: "15115" uid: a3abc8ef-5a38-4ebc-bdbb-1c16bf6e82a0 kind: Event lastTimestamp: "2026-05-25T13:16:01Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T13:16:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:16:01Z" name: success-200-isvc-339a4-predictor-58f8586498-9r8dr.18b2d15401ed8357 namespace: kserve-ci-e2e-test resourceVersion: "15168" uid: 3f9bf295-4b7f-496c-b3fe-a5f06004b184 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-339a4-predictor-58f8586498-9r8dr namespace: kserve-ci-e2e-test resourceVersion: "15115" uid: a3abc8ef-5a38-4ebc-bdbb-1c16bf6e82a0 kind: Event lastTimestamp: "2026-05-25T13:16:01Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-25T13:16:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:16:01Z" name: success-200-isvc-339a4-predictor-58f8586498-9r8dr.18b2d15401f8cdd8 namespace: kserve-ci-e2e-test resourceVersion: "15170" uid: 6f0a4ec6-8240-407b-85aa-624c36e0ee80 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-339a4-predictor-58f8586498-9r8dr namespace: kserve-ci-e2e-test resourceVersion: "15115" uid: a3abc8ef-5a38-4ebc-bdbb-1c16bf6e82a0 kind: Event lastTimestamp: "2026-05-25T13:16:01Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T13:16:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:16:01Z" name: success-200-isvc-339a4-predictor-58f8586498-9r8dr.18b2d15404a53abe namespace: kserve-ci-e2e-test resourceVersion: "15171" uid: e447e85e-8c93-4ea9-9c47-b937cbe6ec54 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-339a4-predictor-58f8586498-9r8dr namespace: kserve-ci-e2e-test resourceVersion: "15115" uid: a3abc8ef-5a38-4ebc-bdbb-1c16bf6e82a0 kind: Event lastTimestamp: "2026-05-25T13:16:01Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T13:16:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:16:01Z" name: success-200-isvc-339a4-predictor-58f8586498-9r8dr.18b2d154051be87f namespace: kserve-ci-e2e-test resourceVersion: "15172" uid: 12fb87c3-79ca-4499-bbbd-ae4b46319eda reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-05-25T13:16:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-339a4-predictor-58f8586498-9r8dr namespace: kserve-ci-e2e-test resourceVersion: "15115" uid: a3abc8ef-5a38-4ebc-bdbb-1c16bf6e82a0 kind: Event lastTimestamp: "2026-05-25T13:25:18Z" message: 'Readiness probe failed: dial tcp 10.134.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T13:16:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:25:18Z" name: success-200-isvc-339a4-predictor-58f8586498-9r8dr.18b2d15447b04eae namespace: kserve-ci-e2e-test resourceVersion: "19573" uid: 6a6e8493-8a84-43e4-b8b8-21e97784c691 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:25:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-339a4-predictor-58f8586498-9r8dr namespace: kserve-ci-e2e-test resourceVersion: "15115" uid: a3abc8ef-5a38-4ebc-bdbb-1c16bf6e82a0 kind: Event lastTimestamp: "2026-05-25T13:25:15Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T13:25:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:25:15Z" name: success-200-isvc-339a4-predictor-58f8586498-9r8dr.18b2d1d50d43f971 namespace: kserve-ci-e2e-test resourceVersion: "19447" uid: c4078da0-ca86-4bff-a149-3936fe52d30e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:25:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-339a4-predictor-58f8586498-9r8dr namespace: kserve-ci-e2e-test resourceVersion: "15115" uid: a3abc8ef-5a38-4ebc-bdbb-1c16bf6e82a0 kind: Event lastTimestamp: "2026-05-25T13:25:15Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T13:25:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:25:15Z" name: success-200-isvc-339a4-predictor-58f8586498-9r8dr.18b2d1d50d448450 namespace: kserve-ci-e2e-test resourceVersion: "19451" uid: 19d9d215-aa34-4da1-b4fd-df867c1d6c39 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:25:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-339a4-predictor-58f8586498-9r8dr namespace: kserve-ci-e2e-test resourceVersion: "15115" uid: a3abc8ef-5a38-4ebc-bdbb-1c16bf6e82a0 kind: Event lastTimestamp: "2026-05-25T13:25:18Z" message: 'Readiness probe failed: Get "https://10.134.0.35:8643/healthz": dial tcp 10.134.0.35:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-25T13:25:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:25:18Z" name: success-200-isvc-339a4-predictor-58f8586498-9r8dr.18b2d1d5bbfd7f9f namespace: kserve-ci-e2e-test resourceVersion: "19572" uid: c4c779a5-a023-4ee1-9ee9-c2e9a866427c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-339a4-predictor-58f8586498 namespace: kserve-ci-e2e-test resourceVersion: "15101" uid: c2d52cbe-6b63-414b-9d8f-68ac7da72c7f kind: Event lastTimestamp: "2026-05-25T13:16:00Z" message: 'Created pod: success-200-isvc-339a4-predictor-58f8586498-9r8dr' metadata: creationTimestamp: "2026-05-25T13:16:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:16:00Z" name: success-200-isvc-339a4-predictor-58f8586498.18b2d153e3281e0f namespace: kserve-ci-e2e-test resourceVersion: "15114" uid: e2f93ab4-fbba-4eda-ab27-3da84609097f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-339a4-predictor namespace: kserve-ci-e2e-test resourceVersion: "15098" uid: ce4b9229-8c71-4fd1-badb-27ce0dc4b49b kind: Event lastTimestamp: "2026-05-25T13:16:00Z" message: Scaled up replica set success-200-isvc-339a4-predictor-58f8586498 from 0 to 1 metadata: creationTimestamp: "2026-05-25T13:16:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:16:00Z" name: success-200-isvc-339a4-predictor.18b2d153e162f7ae namespace: kserve-ci-e2e-test resourceVersion: "15102" uid: 348c5f56-c09d-4bd5-9618-88eb38cfd46b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-339a4-predictor namespace: kserve-ci-e2e-test resourceVersion: "15100" uid: 97f988e5-7d0e-4e77-91ee-186ca4995fbf kind: Event lastTimestamp: "2026-05-25T13:16:15Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T13:16:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:16:15Z" name: success-200-isvc-339a4-predictor.18b2d157607cd3dc namespace: kserve-ci-e2e-test resourceVersion: "15324" uid: 51cdf6d2-e1cf-4c89-940d-a7fe4e1ec5ff reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-339a4-predictor namespace: kserve-ci-e2e-test resourceVersion: "15100" uid: 97f988e5-7d0e-4e77-91ee-186ca4995fbf kind: Event lastTimestamp: "2026-05-25T13:16:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T13:16:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:16:15Z" name: success-200-isvc-339a4-predictor.18b2d15760e42c74 namespace: kserve-ci-e2e-test resourceVersion: "15328" uid: 62f6b34a-55ab-4acd-bd3b-5703dbae42d0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-25T13:16:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-339a4-predictor namespace: kserve-ci-e2e-test resourceVersion: "15325" uid: 97f988e5-7d0e-4e77-91ee-186ca4995fbf kind: Event lastTimestamp: "2026-05-25T13:17:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T13:16:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:17:16Z" name: success-200-isvc-339a4-predictor.18b2d15ae0a045fb namespace: kserve-ci-e2e-test resourceVersion: "16085" uid: 9a60779a-3a05-4b9a-bc29-5da1a8024cd6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-25T13:16:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-339a4-predictor namespace: kserve-ci-e2e-test resourceVersion: "15325" uid: 97f988e5-7d0e-4e77-91ee-186ca4995fbf kind: Event lastTimestamp: "2026-05-25T13:17:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T13:16:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:17:16Z" name: success-200-isvc-339a4-predictor.18b2d15ae1611481 namespace: kserve-ci-e2e-test resourceVersion: "16087" uid: 95e9dcdc-fa92-4603-8bd1-b6ac0b01cd84 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T13:16:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-339a4 namespace: kserve-ci-e2e-test resourceVersion: "15075" uid: 7f65a0de-467d-4af9-850e-3c030a395ebd kind: Event lastTimestamp: "2026-05-25T13:16:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T13:16:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:16:48Z" name: success-200-isvc-339a4.18b2d153e16a69ab namespace: kserve-ci-e2e-test resourceVersion: "15821" uid: 709e30b7-c6f8-4867-b93d-33a3435ec227 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-339a4 namespace: kserve-ci-e2e-test resourceVersion: "15075" uid: 7f65a0de-467d-4af9-850e-3c030a395ebd kind: Event lastTimestamp: "2026-05-25T13:16:00Z" message: 'Failed to update status for InferenceService "success-200-isvc-339a4": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-339a4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T13:16:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:16:00Z" name: success-200-isvc-339a4.18b2d153e1d7380c namespace: kserve-ci-e2e-test resourceVersion: "15107" uid: 854385ae-1386-4c69-9d28-994be54fe383 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-339a4 namespace: kserve-ci-e2e-test resourceVersion: "15075" uid: 7f65a0de-467d-4af9-850e-3c030a395ebd kind: Event lastTimestamp: "2026-05-25T13:16:00Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-339a4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T13:16:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:16:00Z" name: success-200-isvc-339a4.18b2d153e1d78b9d namespace: kserve-ci-e2e-test resourceVersion: "15110" uid: 62b74e2c-183e-49e3-a3f4-95a39b44799b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-339a4 namespace: kserve-ci-e2e-test resourceVersion: "15819" uid: 7f65a0de-467d-4af9-850e-3c030a395ebd kind: Event lastTimestamp: "2026-05-25T13:16:48Z" message: InferenceService [success-200-isvc-339a4] is Ready metadata: creationTimestamp: "2026-05-25T13:16:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:16:48Z" name: success-200-isvc-339a4.18b2d15f02fd9d46 namespace: kserve-ci-e2e-test resourceVersion: "15820" uid: 629c2d80-528e-4e7d-b148-b30f89a1d87e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T13:35:41.120101Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-35687-predictor-6d599c8c48-68thq namespace: kserve-ci-e2e-test resourceVersion: "24731" uid: 1fbb8b99-22aa-4a70-87f2-e46d72c808f6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-35687-predictor-6d599c8c48-68thq to ip-10-0-135-103.ec2.internal metadata: creationTimestamp: "2026-05-25T13:35:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T13:35:41Z" name: success-200-isvc-35687-predictor-6d599c8c48-68thq.18b2d266a87e432c namespace: kserve-ci-e2e-test resourceVersion: "24741" uid: 6b29b154-76bc-462e-bf0c-9e9ff1a2f84f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c46b5f76-4gnkb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:35:41Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-35687-predictor-6d599c8c48-68thq namespace: kserve-ci-e2e-test resourceVersion: "24752" uid: 1fbb8b99-22aa-4a70-87f2-e46d72c808f6 kind: Event lastTimestamp: "2026-05-25T13:35:41Z" message: Add eth0 [10.134.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T13:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T13:35:41Z" name: success-200-isvc-35687-predictor-6d599c8c48-68thq.18b2d266c25c2680 namespace: kserve-ci-e2e-test resourceVersion: "24775" uid: 66a8eec4-3b66-48c5-9aee-aac81252c6ee reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:35:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-35687-predictor-6d599c8c48-68thq namespace: kserve-ci-e2e-test resourceVersion: "24735" uid: 1fbb8b99-22aa-4a70-87f2-e46d72c808f6 kind: Event lastTimestamp: "2026-05-25T13:35:41Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-25T13:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:35:41Z" name: success-200-isvc-35687-predictor-6d599c8c48-68thq.18b2d266c3c60ef5 namespace: kserve-ci-e2e-test resourceVersion: "24777" uid: aeaa6a5f-05ae-49bb-b629-5cf5e503491a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:35:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-35687-predictor-6d599c8c48-68thq namespace: kserve-ci-e2e-test resourceVersion: "24735" uid: 1fbb8b99-22aa-4a70-87f2-e46d72c808f6 kind: Event lastTimestamp: "2026-05-25T13:35:41Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T13:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:35:41Z" name: success-200-isvc-35687-predictor-6d599c8c48-68thq.18b2d266c749fdcc namespace: kserve-ci-e2e-test resourceVersion: "24780" uid: 2facbad3-7a19-403f-95b5-d5a279450222 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:35:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-35687-predictor-6d599c8c48-68thq namespace: kserve-ci-e2e-test resourceVersion: "24735" uid: 1fbb8b99-22aa-4a70-87f2-e46d72c808f6 kind: Event lastTimestamp: "2026-05-25T13:35:41Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T13:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:35:41Z" name: success-200-isvc-35687-predictor-6d599c8c48-68thq.18b2d266c7d19ec2 namespace: kserve-ci-e2e-test resourceVersion: "24782" uid: 7bbb826c-73cb-45a6-830e-a7bc953cd9c8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:35:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-35687-predictor-6d599c8c48-68thq namespace: kserve-ci-e2e-test resourceVersion: "24735" uid: 1fbb8b99-22aa-4a70-87f2-e46d72c808f6 kind: Event lastTimestamp: "2026-05-25T13:35:41Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-25T13:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:35:41Z" name: success-200-isvc-35687-predictor-6d599c8c48-68thq.18b2d266c7dda8db namespace: kserve-ci-e2e-test resourceVersion: "24783" uid: a8de8cd4-89d6-4e36-a071-b9f317960f52 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:35:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-35687-predictor-6d599c8c48-68thq namespace: kserve-ci-e2e-test resourceVersion: "24735" uid: 1fbb8b99-22aa-4a70-87f2-e46d72c808f6 kind: Event lastTimestamp: "2026-05-25T13:35:41Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T13:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:35:41Z" name: success-200-isvc-35687-predictor-6d599c8c48-68thq.18b2d266ca993a3a namespace: kserve-ci-e2e-test resourceVersion: "24785" uid: 183be3ec-86e6-4e1f-931c-77fb16eebf63 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:35:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-35687-predictor-6d599c8c48-68thq namespace: kserve-ci-e2e-test resourceVersion: "24735" uid: 1fbb8b99-22aa-4a70-87f2-e46d72c808f6 kind: Event lastTimestamp: "2026-05-25T13:35:41Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T13:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:35:41Z" name: success-200-isvc-35687-predictor-6d599c8c48-68thq.18b2d266cb22b5ce namespace: kserve-ci-e2e-test resourceVersion: "24787" uid: a79523e3-f212-4b6b-8e79-04ee202df7da reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T13:35:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-35687-predictor-6d599c8c48-68thq namespace: kserve-ci-e2e-test resourceVersion: "24735" uid: 1fbb8b99-22aa-4a70-87f2-e46d72c808f6 kind: Event lastTimestamp: "2026-05-25T13:36:18Z" message: 'Readiness probe failed: dial tcp 10.134.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T13:35:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:36:18Z" name: success-200-isvc-35687-predictor-6d599c8c48-68thq.18b2d26707e92146 namespace: kserve-ci-e2e-test resourceVersion: "25244" uid: 1aa356c1-2f05-4246-b270-9772d1e30bb2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-35687-predictor-6d599c8c48-68thq namespace: kserve-ci-e2e-test resourceVersion: "24735" uid: 1fbb8b99-22aa-4a70-87f2-e46d72c808f6 kind: Event lastTimestamp: "2026-05-25T13:36:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T13:36:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:36:51Z" name: success-200-isvc-35687-predictor-6d599c8c48-68thq.18b2d2770160027c namespace: kserve-ci-e2e-test resourceVersion: "25628" uid: 1c137120-468e-4bb5-ba1a-265d9cce5438 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-35687-predictor-6d599c8c48-68thq namespace: kserve-ci-e2e-test resourceVersion: "24735" uid: 1fbb8b99-22aa-4a70-87f2-e46d72c808f6 kind: Event lastTimestamp: "2026-05-25T13:36:51Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T13:36:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:36:51Z" name: success-200-isvc-35687-predictor-6d599c8c48-68thq.18b2d2770160af6f namespace: kserve-ci-e2e-test resourceVersion: "25630" uid: cf03a947-9e9e-4654-a74f-fef6eb57b79a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-35687-predictor-6d599c8c48-68thq namespace: kserve-ci-e2e-test resourceVersion: "24735" uid: 1fbb8b99-22aa-4a70-87f2-e46d72c808f6 kind: Event lastTimestamp: "2026-05-25T13:36:53Z" message: 'Readiness probe failed: Get "https://10.134.0.41:8643/healthz": dial tcp 10.134.0.41:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-25T13:36:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:36:53Z" name: success-200-isvc-35687-predictor-6d599c8c48-68thq.18b2d2779002b5d9 namespace: kserve-ci-e2e-test resourceVersion: "25718" uid: ec60ed72-e307-4e00-affb-ab04a0b89190 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:35:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-35687-predictor-6d599c8c48 namespace: kserve-ci-e2e-test resourceVersion: "24700" uid: 95af8eea-a38d-4df2-becc-9027349fa2f1 kind: Event lastTimestamp: "2026-05-25T13:35:41Z" message: 'Created pod: success-200-isvc-35687-predictor-6d599c8c48-68thq' metadata: creationTimestamp: "2026-05-25T13:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:35:41Z" name: success-200-isvc-35687-predictor-6d599c8c48.18b2d266a7e29322 namespace: kserve-ci-e2e-test resourceVersion: "24739" uid: c4955e17-ce2d-44a7-9114-f461200238f2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:35:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-35687-predictor namespace: kserve-ci-e2e-test resourceVersion: "24697" uid: 40025eaa-9068-4fac-b40e-7f5dad109ee1 kind: Event lastTimestamp: "2026-05-25T13:35:41Z" message: Scaled up replica set success-200-isvc-35687-predictor-6d599c8c48 from 0 to 1 metadata: creationTimestamp: "2026-05-25T13:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:35:41Z" name: success-200-isvc-35687-predictor.18b2d266a22d5b51 namespace: kserve-ci-e2e-test resourceVersion: "24704" uid: 2f4e40f6-60ee-4325-978f-16955b9d49f8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T13:35:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-35687-predictor namespace: kserve-ci-e2e-test resourceVersion: "24699" uid: 6017e6ca-5dd0-4f3d-9183-a343ab956657 kind: Event lastTimestamp: "2026-05-25T13:36:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T13:35:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:36:41Z" name: success-200-isvc-35687-predictor.18b2d26a20bbc85a namespace: kserve-ci-e2e-test resourceVersion: "25455" uid: 5c5b5b97-3ea2-413d-82b3-652f45355abb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T13:35:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-35687-predictor namespace: kserve-ci-e2e-test resourceVersion: "24699" uid: 6017e6ca-5dd0-4f3d-9183-a343ab956657 kind: Event lastTimestamp: "2026-05-25T13:36:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T13:35:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:36:41Z" name: success-200-isvc-35687-predictor.18b2d26a2119e27f namespace: kserve-ci-e2e-test resourceVersion: "25458" uid: e8e99f3a-521e-4212-850a-a5ad8ce59155 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-35687-predictor namespace: kserve-ci-e2e-test resourceVersion: "24915" uid: 6017e6ca-5dd0-4f3d-9183-a343ab956657 kind: Event lastTimestamp: "2026-05-25T13:36:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T13:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:36:11Z" name: success-200-isvc-35687-predictor.18b2d26da036a82e namespace: kserve-ci-e2e-test resourceVersion: "25072" uid: 1efb01d1-3d30-488f-9ac4-cfb999f527ce reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-35687-predictor namespace: kserve-ci-e2e-test resourceVersion: "24915" uid: 6017e6ca-5dd0-4f3d-9183-a343ab956657 kind: Event lastTimestamp: "2026-05-25T13:36:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T13:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:36:11Z" name: success-200-isvc-35687-predictor.18b2d26da0b36617 namespace: kserve-ci-e2e-test resourceVersion: "25078" uid: 82f6866f-71c1-4269-965c-457b0da7dea8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T13:35:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-35687 namespace: kserve-ci-e2e-test resourceVersion: "24677" uid: f3b624ce-99d2-419f-8d4b-4ed7a19241a3 kind: Event lastTimestamp: "2026-05-25T13:36:28Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T13:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:36:28Z" name: success-200-isvc-35687.18b2d266a1fd8282 namespace: kserve-ci-e2e-test resourceVersion: "25339" uid: 35ff7818-2a43-4226-9bf5-544f9c9f1993 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:35:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-35687 namespace: kserve-ci-e2e-test resourceVersion: "24677" uid: f3b624ce-99d2-419f-8d4b-4ed7a19241a3 kind: Event lastTimestamp: "2026-05-25T13:35:41Z" message: 'Failed to update status for InferenceService "success-200-isvc-35687": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-35687": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T13:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:35:41Z" name: success-200-isvc-35687.18b2d266a26709dc namespace: kserve-ci-e2e-test resourceVersion: "24706" uid: 8b9a8959-f679-49f2-9eb4-6dfc7320995d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:35:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-35687 namespace: kserve-ci-e2e-test resourceVersion: "24677" uid: f3b624ce-99d2-419f-8d4b-4ed7a19241a3 kind: Event lastTimestamp: "2026-05-25T13:35:41Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-35687": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T13:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:35:41Z" name: success-200-isvc-35687.18b2d266a2676079 namespace: kserve-ci-e2e-test resourceVersion: "24707" uid: 6fb770e1-21a6-4a92-aa91-24b11530de19 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-35687 namespace: kserve-ci-e2e-test resourceVersion: "25337" uid: f3b624ce-99d2-419f-8d4b-4ed7a19241a3 kind: Event lastTimestamp: "2026-05-25T13:36:28Z" message: InferenceService [success-200-isvc-35687] is Ready metadata: creationTimestamp: "2026-05-25T13:36:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:36:28Z" name: success-200-isvc-35687.18b2d271c29293d8 namespace: kserve-ci-e2e-test resourceVersion: "25338" uid: e191c6b6-0d08-412d-8d10-4574baa3488d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T13:26:26.181088Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3e647-predictor-55bfd66d8d-pj8fl namespace: kserve-ci-e2e-test resourceVersion: "20358" uid: ade8fe09-bea0-4b3d-a568-e52c3fa9fe0d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-3e647-predictor-55bfd66d8d-pj8fl to ip-10-0-135-103.ec2.internal metadata: creationTimestamp: "2026-05-25T13:26:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T13:26:26Z" name: success-200-isvc-3e647-predictor-55bfd66d8d-pj8fl.18b2d1e5738cd7f8 namespace: kserve-ci-e2e-test resourceVersion: "20363" uid: 8fb64092-9e4c-4dd4-8ab4-9c1036e862f8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c46b5f76-4gnkb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:26Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3e647-predictor-55bfd66d8d-pj8fl namespace: kserve-ci-e2e-test resourceVersion: "20360" uid: ade8fe09-bea0-4b3d-a568-e52c3fa9fe0d kind: Event lastTimestamp: "2026-05-25T13:26:26Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-3e647-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-25T13:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:26:26Z" name: success-200-isvc-3e647-predictor-55bfd66d8d-pj8fl.18b2d1e58001ce32 namespace: kserve-ci-e2e-test resourceVersion: "20412" uid: c299add0-bdce-45b1-94c9-edae1421f315 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:27Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3e647-predictor-55bfd66d8d-pj8fl namespace: kserve-ci-e2e-test resourceVersion: "20378" uid: ade8fe09-bea0-4b3d-a568-e52c3fa9fe0d kind: Event lastTimestamp: "2026-05-25T13:26:27Z" message: Add eth0 [10.134.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T13:26:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T13:26:27Z" name: success-200-isvc-3e647-predictor-55bfd66d8d-pj8fl.18b2d1e5b0b7d25b namespace: kserve-ci-e2e-test resourceVersion: "20436" uid: d6afbe4e-67fb-406b-b13d-8abd6ff3f7f5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3e647-predictor-55bfd66d8d-pj8fl namespace: kserve-ci-e2e-test resourceVersion: "20360" uid: ade8fe09-bea0-4b3d-a568-e52c3fa9fe0d kind: Event lastTimestamp: "2026-05-25T13:26:27Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-25T13:26:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:26:27Z" name: success-200-isvc-3e647-predictor-55bfd66d8d-pj8fl.18b2d1e5b232d749 namespace: kserve-ci-e2e-test resourceVersion: "20439" uid: 4785aefb-1158-4dc8-89e8-a2c084bc54c9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3e647-predictor-55bfd66d8d-pj8fl namespace: kserve-ci-e2e-test resourceVersion: "20360" uid: ade8fe09-bea0-4b3d-a568-e52c3fa9fe0d kind: Event lastTimestamp: "2026-05-25T13:26:27Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T13:26:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:26:27Z" name: success-200-isvc-3e647-predictor-55bfd66d8d-pj8fl.18b2d1e5b5e08adf namespace: kserve-ci-e2e-test resourceVersion: "20440" uid: 191aa26e-fbba-428d-a955-0936f1e6b3c4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3e647-predictor-55bfd66d8d-pj8fl namespace: kserve-ci-e2e-test resourceVersion: "20360" uid: ade8fe09-bea0-4b3d-a568-e52c3fa9fe0d kind: Event lastTimestamp: "2026-05-25T13:26:27Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T13:26:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:26:27Z" name: success-200-isvc-3e647-predictor-55bfd66d8d-pj8fl.18b2d1e5b658b867 namespace: kserve-ci-e2e-test resourceVersion: "20441" uid: 9b9fe982-e113-4fef-bea7-40f4216bf3a6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3e647-predictor-55bfd66d8d-pj8fl namespace: kserve-ci-e2e-test resourceVersion: "20360" uid: ade8fe09-bea0-4b3d-a568-e52c3fa9fe0d kind: Event lastTimestamp: "2026-05-25T13:26:27Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-25T13:26:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:26:27Z" name: success-200-isvc-3e647-predictor-55bfd66d8d-pj8fl.18b2d1e5b665a3b8 namespace: kserve-ci-e2e-test resourceVersion: "20442" uid: 8f74cf88-34a2-4c9b-a2be-97cfd8614c02 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3e647-predictor-55bfd66d8d-pj8fl namespace: kserve-ci-e2e-test resourceVersion: "20360" uid: ade8fe09-bea0-4b3d-a568-e52c3fa9fe0d kind: Event lastTimestamp: "2026-05-25T13:26:27Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T13:26:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:26:27Z" name: success-200-isvc-3e647-predictor-55bfd66d8d-pj8fl.18b2d1e5b9c2a349 namespace: kserve-ci-e2e-test resourceVersion: "20444" uid: 49f5f455-6bb7-4b78-b634-2e9039f20275 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3e647-predictor-55bfd66d8d-pj8fl namespace: kserve-ci-e2e-test resourceVersion: "20360" uid: ade8fe09-bea0-4b3d-a568-e52c3fa9fe0d kind: Event lastTimestamp: "2026-05-25T13:26:27Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T13:26:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:26:27Z" name: success-200-isvc-3e647-predictor-55bfd66d8d-pj8fl.18b2d1e5ba391ebc namespace: kserve-ci-e2e-test resourceVersion: "20445" uid: 2797d65f-ab90-4666-b79d-5e5470e67902 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T13:26:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3e647-predictor-55bfd66d8d-pj8fl namespace: kserve-ci-e2e-test resourceVersion: "20360" uid: ade8fe09-bea0-4b3d-a568-e52c3fa9fe0d kind: Event lastTimestamp: "2026-05-25T13:27:04Z" message: 'Readiness probe failed: dial tcp 10.134.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T13:26:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:27:04Z" name: success-200-isvc-3e647-predictor-55bfd66d8d-pj8fl.18b2d1e6123c9ea5 namespace: kserve-ci-e2e-test resourceVersion: "20968" uid: 2cc19830-a8cc-4da6-ba1a-14d60e6718f0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:35:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3e647-predictor-55bfd66d8d-pj8fl namespace: kserve-ci-e2e-test resourceVersion: "20360" uid: ade8fe09-bea0-4b3d-a568-e52c3fa9fe0d kind: Event lastTimestamp: "2026-05-25T13:35:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T13:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:35:41Z" name: success-200-isvc-3e647-predictor-55bfd66d8d-pj8fl.18b2d266a1c3be3e namespace: kserve-ci-e2e-test resourceVersion: "24701" uid: 1a2daeb6-1c1c-4361-8334-169415d3d1c0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:35:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3e647-predictor-55bfd66d8d-pj8fl namespace: kserve-ci-e2e-test resourceVersion: "20360" uid: ade8fe09-bea0-4b3d-a568-e52c3fa9fe0d kind: Event lastTimestamp: "2026-05-25T13:35:41Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T13:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:35:41Z" name: success-200-isvc-3e647-predictor-55bfd66d8d-pj8fl.18b2d266a1c45616 namespace: kserve-ci-e2e-test resourceVersion: "24703" uid: 93b3e1ef-6fb4-483b-bec6-b14b26b3585b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-3e647-predictor-55bfd66d8d namespace: kserve-ci-e2e-test resourceVersion: "20347" uid: dc3e8f25-e104-48a7-b822-eb4115e0a87e kind: Event lastTimestamp: "2026-05-25T13:26:26Z" message: 'Created pod: success-200-isvc-3e647-predictor-55bfd66d8d-pj8fl' metadata: creationTimestamp: "2026-05-25T13:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:26:26Z" name: success-200-isvc-3e647-predictor-55bfd66d8d.18b2d1e572f1a78d namespace: kserve-ci-e2e-test resourceVersion: "20359" uid: b500d94e-2cdc-401c-a8e1-eca5560ef51e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-3e647-predictor namespace: kserve-ci-e2e-test resourceVersion: "20342" uid: 12edccb5-5ac2-4a43-8f5c-6576266b3c20 kind: Event lastTimestamp: "2026-05-25T13:26:26Z" message: Scaled up replica set success-200-isvc-3e647-predictor-55bfd66d8d from 0 to 1 metadata: creationTimestamp: "2026-05-25T13:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:26:26Z" name: success-200-isvc-3e647-predictor.18b2d1e570da3403 namespace: kserve-ci-e2e-test resourceVersion: "20350" uid: 4280ab1f-d85a-48e1-a2d9-cf6e8be6802e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T13:26:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3e647-predictor namespace: kserve-ci-e2e-test resourceVersion: "20346" uid: a9ddca4d-84ca-4947-b720-a77bb0097859 kind: Event lastTimestamp: "2026-05-25T13:27:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T13:26:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:27:11Z" name: success-200-isvc-3e647-predictor.18b2d1e8efa43650 namespace: kserve-ci-e2e-test resourceVersion: "21018" uid: 7ad40a2f-e355-417b-a837-eb36ccb0591d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T13:26:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3e647-predictor namespace: kserve-ci-e2e-test resourceVersion: "20346" uid: a9ddca4d-84ca-4947-b720-a77bb0097859 kind: Event lastTimestamp: "2026-05-25T13:27:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T13:26:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:27:11Z" name: success-200-isvc-3e647-predictor.18b2d1e8f00d590c namespace: kserve-ci-e2e-test resourceVersion: "21019" uid: a5260910-b6ad-450c-adad-844e32a1e24e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T13:27:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3e647-predictor namespace: kserve-ci-e2e-test resourceVersion: "20587" uid: a9ddca4d-84ca-4947-b720-a77bb0097859 kind: Event lastTimestamp: "2026-05-25T13:27:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T13:27:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:27:41Z" name: success-200-isvc-3e647-predictor.18b2d1f36ca1fca8 namespace: kserve-ci-e2e-test resourceVersion: "21310" uid: 0701a427-12e7-4eb5-9b70-982030ce0753 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T13:27:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3e647-predictor namespace: kserve-ci-e2e-test resourceVersion: "20587" uid: a9ddca4d-84ca-4947-b720-a77bb0097859 kind: Event lastTimestamp: "2026-05-25T13:27:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T13:27:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:27:41Z" name: success-200-isvc-3e647-predictor.18b2d1f36d11428f namespace: kserve-ci-e2e-test resourceVersion: "21311" uid: bf244211-5b56-4711-9d57-ade42272ef72 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T13:26:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3e647 namespace: kserve-ci-e2e-test resourceVersion: "20326" uid: 70104cca-105b-4376-9c0c-d5f3e6d913e0 kind: Event lastTimestamp: "2026-05-25T13:27:14Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T13:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:27:14Z" name: success-200-isvc-3e647.18b2d1e570a3b036 namespace: kserve-ci-e2e-test resourceVersion: "21061" uid: 674aebd0-8c5d-4938-b68f-187f101b70f1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3e647 namespace: kserve-ci-e2e-test resourceVersion: "20326" uid: 70104cca-105b-4376-9c0c-d5f3e6d913e0 kind: Event lastTimestamp: "2026-05-25T13:26:26Z" message: 'Failed to update status for InferenceService "success-200-isvc-3e647": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-3e647": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T13:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:26:26Z" name: success-200-isvc-3e647.18b2d1e5712e0252 namespace: kserve-ci-e2e-test resourceVersion: "20352" uid: d74e830c-441d-4409-9af3-795ed6da1cd2 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3e647 namespace: kserve-ci-e2e-test resourceVersion: "20326" uid: 70104cca-105b-4376-9c0c-d5f3e6d913e0 kind: Event lastTimestamp: "2026-05-25T13:26:26Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-3e647": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T13:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:26:26Z" name: success-200-isvc-3e647.18b2d1e5712e4f96 namespace: kserve-ci-e2e-test resourceVersion: "20354" uid: 873d044a-5827-4aaa-92cc-27e16c856450 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:27:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3e647 namespace: kserve-ci-e2e-test resourceVersion: "21059" uid: 70104cca-105b-4376-9c0c-d5f3e6d913e0 kind: Event lastTimestamp: "2026-05-25T13:27:14Z" message: InferenceService [success-200-isvc-3e647] is Ready metadata: creationTimestamp: "2026-05-25T13:27:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:27:14Z" name: success-200-isvc-3e647.18b2d1f0ccca1dd4 namespace: kserve-ci-e2e-test resourceVersion: "21060" uid: b1b9a463-452e-4eb0-8031-7784d9c77382 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T13:26:56.212266Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-72c4b-predictor-6f6fcbb-zg7jb namespace: kserve-ci-e2e-test resourceVersion: "20814" uid: 31d66af3-84e0-4725-a704-7d9532b0ffa9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-72c4b-predictor-6f6fcbb-zg7jb to ip-10-0-135-103.ec2.internal metadata: creationTimestamp: "2026-05-25T13:26:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T13:26:56Z" name: success-200-isvc-72c4b-predictor-6f6fcbb-zg7jb.18b2d1ec718c5137 namespace: kserve-ci-e2e-test resourceVersion: "20819" uid: ec15846a-4edc-4968-b928-ab41d0a319cd reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c46b5f76-4gnkb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:56Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-72c4b-predictor-6f6fcbb-zg7jb namespace: kserve-ci-e2e-test resourceVersion: "20816" uid: 31d66af3-84e0-4725-a704-7d9532b0ffa9 kind: Event lastTimestamp: "2026-05-25T13:26:56Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-72c4b-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-25T13:26:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:26:56Z" name: success-200-isvc-72c4b-predictor-6f6fcbb-zg7jb.18b2d1ec7fb00214 namespace: kserve-ci-e2e-test resourceVersion: "20854" uid: c57f8042-6283-4c43-9c18-ee16390a7a44 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:57Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-72c4b-predictor-6f6fcbb-zg7jb namespace: kserve-ci-e2e-test resourceVersion: "20824" uid: 31d66af3-84e0-4725-a704-7d9532b0ffa9 kind: Event lastTimestamp: "2026-05-25T13:26:57Z" message: Add eth0 [10.134.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T13:26:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T13:26:57Z" name: success-200-isvc-72c4b-predictor-6f6fcbb-zg7jb.18b2d1ecaee5943c namespace: kserve-ci-e2e-test resourceVersion: "20881" uid: 4bc176c6-97eb-4813-aecc-e5328b6fc6a5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-72c4b-predictor-6f6fcbb-zg7jb namespace: kserve-ci-e2e-test resourceVersion: "20816" uid: 31d66af3-84e0-4725-a704-7d9532b0ffa9 kind: Event lastTimestamp: "2026-05-25T13:26:57Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-25T13:26:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:26:57Z" name: success-200-isvc-72c4b-predictor-6f6fcbb-zg7jb.18b2d1ecb02fbe6c namespace: kserve-ci-e2e-test resourceVersion: "20883" uid: 9225fb5e-86cd-4b41-89b9-7d4bfd4a7016 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-72c4b-predictor-6f6fcbb-zg7jb namespace: kserve-ci-e2e-test resourceVersion: "20816" uid: 31d66af3-84e0-4725-a704-7d9532b0ffa9 kind: Event lastTimestamp: "2026-05-25T13:26:57Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T13:26:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:26:57Z" name: success-200-isvc-72c4b-predictor-6f6fcbb-zg7jb.18b2d1ecb45791bb namespace: kserve-ci-e2e-test resourceVersion: "20884" uid: f3739092-244e-4595-809b-a4a160fc8e7e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-72c4b-predictor-6f6fcbb-zg7jb namespace: kserve-ci-e2e-test resourceVersion: "20816" uid: 31d66af3-84e0-4725-a704-7d9532b0ffa9 kind: Event lastTimestamp: "2026-05-25T13:26:57Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T13:26:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:26:57Z" name: success-200-isvc-72c4b-predictor-6f6fcbb-zg7jb.18b2d1ecb4d1845a namespace: kserve-ci-e2e-test resourceVersion: "20885" uid: 8d5a1ca8-a2f9-4454-a953-5209e5112668 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-72c4b-predictor-6f6fcbb-zg7jb namespace: kserve-ci-e2e-test resourceVersion: "20816" uid: 31d66af3-84e0-4725-a704-7d9532b0ffa9 kind: Event lastTimestamp: "2026-05-25T13:26:57Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-25T13:26:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:26:57Z" name: success-200-isvc-72c4b-predictor-6f6fcbb-zg7jb.18b2d1ecb4de4364 namespace: kserve-ci-e2e-test resourceVersion: "20886" uid: d7735462-5e75-4139-8683-04f78debae63 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-72c4b-predictor-6f6fcbb-zg7jb namespace: kserve-ci-e2e-test resourceVersion: "20816" uid: 31d66af3-84e0-4725-a704-7d9532b0ffa9 kind: Event lastTimestamp: "2026-05-25T13:26:57Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T13:26:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:26:57Z" name: success-200-isvc-72c4b-predictor-6f6fcbb-zg7jb.18b2d1ecb81bc50e namespace: kserve-ci-e2e-test resourceVersion: "20887" uid: 57ac18dc-ed75-4aa3-a45b-4caa0c817802 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-72c4b-predictor-6f6fcbb-zg7jb namespace: kserve-ci-e2e-test resourceVersion: "20816" uid: 31d66af3-84e0-4725-a704-7d9532b0ffa9 kind: Event lastTimestamp: "2026-05-25T13:26:57Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T13:26:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:26:57Z" name: success-200-isvc-72c4b-predictor-6f6fcbb-zg7jb.18b2d1ecb88fca89 namespace: kserve-ci-e2e-test resourceVersion: "20888" uid: 4ae6e127-da31-4d19-afd9-01ec67701a2e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-05-25T13:26:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-72c4b-predictor-6f6fcbb-zg7jb namespace: kserve-ci-e2e-test resourceVersion: "20816" uid: 31d66af3-84e0-4725-a704-7d9532b0ffa9 kind: Event lastTimestamp: "2026-05-25T13:36:13Z" message: 'Readiness probe failed: dial tcp 10.134.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T13:26:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:36:13Z" name: success-200-isvc-72c4b-predictor-6f6fcbb-zg7jb.18b2d1ecd94841b9 namespace: kserve-ci-e2e-test resourceVersion: "25200" uid: 6bcb6ec4-0f88-4c9d-8415-00fccf5e70ce reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-72c4b-predictor-6f6fcbb-zg7jb namespace: kserve-ci-e2e-test resourceVersion: "20816" uid: 31d66af3-84e0-4725-a704-7d9532b0ffa9 kind: Event lastTimestamp: "2026-05-25T13:36:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T13:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:36:11Z" name: success-200-isvc-72c4b-predictor-6f6fcbb-zg7jb.18b2d26da119ced6 namespace: kserve-ci-e2e-test resourceVersion: "25080" uid: 687f7a6d-4f85-485a-bae8-a78cb8dcf855 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-72c4b-predictor-6f6fcbb-zg7jb namespace: kserve-ci-e2e-test resourceVersion: "20816" uid: 31d66af3-84e0-4725-a704-7d9532b0ffa9 kind: Event lastTimestamp: "2026-05-25T13:36:11Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T13:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:36:11Z" name: success-200-isvc-72c4b-predictor-6f6fcbb-zg7jb.18b2d26da11a5178 namespace: kserve-ci-e2e-test resourceVersion: "25084" uid: 41e8d74b-159e-4cb1-a938-094115f839ca reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-72c4b-predictor-6f6fcbb-zg7jb namespace: kserve-ci-e2e-test resourceVersion: "20816" uid: 31d66af3-84e0-4725-a704-7d9532b0ffa9 kind: Event lastTimestamp: "2026-05-25T13:36:13Z" message: 'Readiness probe failed: Get "https://10.134.0.40:8643/healthz": dial tcp 10.134.0.40:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-25T13:36:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:36:13Z" name: success-200-isvc-72c4b-predictor-6f6fcbb-zg7jb.18b2d26e4d96c68a namespace: kserve-ci-e2e-test resourceVersion: "25199" uid: 7a6e54c6-c190-44e6-84f6-fb2ec96d0594 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-72c4b-predictor-6f6fcbb namespace: kserve-ci-e2e-test resourceVersion: "20806" uid: fad74400-1e37-4a31-b383-ef189cc16cdc kind: Event lastTimestamp: "2026-05-25T13:26:56Z" message: 'Created pod: success-200-isvc-72c4b-predictor-6f6fcbb-zg7jb' metadata: creationTimestamp: "2026-05-25T13:26:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:26:56Z" name: success-200-isvc-72c4b-predictor-6f6fcbb.18b2d1ec712f9b98 namespace: kserve-ci-e2e-test resourceVersion: "20817" uid: 111561e0-2c68-41ca-88a9-35ea76480f2a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-72c4b-predictor namespace: kserve-ci-e2e-test resourceVersion: "20802" uid: f0b2b093-aa23-4f8c-9d8d-e1bc61e380df kind: Event lastTimestamp: "2026-05-25T13:26:56Z" message: Scaled up replica set success-200-isvc-72c4b-predictor-6f6fcbb from 0 to 1 metadata: creationTimestamp: "2026-05-25T13:26:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:26:56Z" name: success-200-isvc-72c4b-predictor.18b2d1ec6f48bfd8 namespace: kserve-ci-e2e-test resourceVersion: "20808" uid: 5d36aed2-6dac-461d-b878-6b4c49c01ffa reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T13:27:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-72c4b-predictor namespace: kserve-ci-e2e-test resourceVersion: "20804" uid: 0ccc8879-6c86-4a70-8d7b-138760583c10 kind: Event lastTimestamp: "2026-05-25T13:27:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T13:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:27:26Z" name: success-200-isvc-72c4b-predictor.18b2d1efedc916fb namespace: kserve-ci-e2e-test resourceVersion: "21158" uid: 69ad9d52-9148-4489-965d-1417ec313c1f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T13:27:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-72c4b-predictor namespace: kserve-ci-e2e-test resourceVersion: "20804" uid: 0ccc8879-6c86-4a70-8d7b-138760583c10 kind: Event lastTimestamp: "2026-05-25T13:27:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T13:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:27:26Z" name: success-200-isvc-72c4b-predictor.18b2d1efee284405 namespace: kserve-ci-e2e-test resourceVersion: "21163" uid: ee00d9f7-85e8-431d-9464-e7044727f133 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T13:27:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-72c4b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21017" uid: 0ccc8879-6c86-4a70-8d7b-138760583c10 kind: Event lastTimestamp: "2026-05-25T13:28:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T13:27:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:28:11Z" name: success-200-isvc-72c4b-predictor.18b2d1f6ebc3b712 namespace: kserve-ci-e2e-test resourceVersion: "21592" uid: 0a06b187-b4ff-4851-8e4e-a3f49520d238 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T13:27:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-72c4b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21017" uid: 0ccc8879-6c86-4a70-8d7b-138760583c10 kind: Event lastTimestamp: "2026-05-25T13:28:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T13:27:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:28:11Z" name: success-200-isvc-72c4b-predictor.18b2d1f6ec3ffb09 namespace: kserve-ci-e2e-test resourceVersion: "21593" uid: 24fa4995-8bfa-46a4-b758-217cbff7fad5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T13:26:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-72c4b namespace: kserve-ci-e2e-test resourceVersion: "20785" uid: 70488c0b-746a-4b94-977a-96e92b9f0716 kind: Event lastTimestamp: "2026-05-25T13:27:44Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T13:26:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:27:44Z" name: success-200-isvc-72c4b.18b2d1ec6f19601d namespace: kserve-ci-e2e-test resourceVersion: "21348" uid: d970b172-ceff-435f-a7e6-d4c0fa03801c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-72c4b namespace: kserve-ci-e2e-test resourceVersion: "20785" uid: 70488c0b-746a-4b94-977a-96e92b9f0716 kind: Event lastTimestamp: "2026-05-25T13:26:56Z" message: 'Failed to update status for InferenceService "success-200-isvc-72c4b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-72c4b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T13:26:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:26:56Z" name: success-200-isvc-72c4b.18b2d1ec6fc40590 namespace: kserve-ci-e2e-test resourceVersion: "20811" uid: ac7d3d7a-af62-421f-b58a-8c52662a864c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-72c4b namespace: kserve-ci-e2e-test resourceVersion: "20785" uid: 70488c0b-746a-4b94-977a-96e92b9f0716 kind: Event lastTimestamp: "2026-05-25T13:26:56Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-72c4b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T13:26:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:26:56Z" name: success-200-isvc-72c4b.18b2d1ec6fc45ecd namespace: kserve-ci-e2e-test resourceVersion: "20812" uid: 17d7e528-10eb-4e64-bac7-23c8dd1d5ae2 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:27:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-72c4b namespace: kserve-ci-e2e-test resourceVersion: "21346" uid: 70488c0b-746a-4b94-977a-96e92b9f0716 kind: Event lastTimestamp: "2026-05-25T13:27:44Z" message: InferenceService [success-200-isvc-72c4b] is Ready metadata: creationTimestamp: "2026-05-25T13:27:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:27:44Z" name: success-200-isvc-72c4b.18b2d1f7941d8508 namespace: kserve-ci-e2e-test resourceVersion: "21347" uid: 8e893530-5d45-4e35-b8f3-e37afe12a80b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T13:36:51.347472Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-766ab-predictor-69d568958b-r85bq namespace: kserve-ci-e2e-test resourceVersion: "25629" uid: 62f285ff-1f15-4a53-992e-f071b91c6c0a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-766ab-predictor-69d568958b-r85bq to ip-10-0-135-103.ec2.internal metadata: creationTimestamp: "2026-05-25T13:36:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T13:36:51Z" name: success-200-isvc-766ab-predictor-69d568958b-r85bq.18b2d277025ed5c2 namespace: kserve-ci-e2e-test resourceVersion: "25634" uid: 910816f0-f51b-4528-8bc8-cab63cb13bf8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c46b5f76-4gnkb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:51Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-766ab-predictor-69d568958b-r85bq namespace: kserve-ci-e2e-test resourceVersion: "25632" uid: 62f285ff-1f15-4a53-992e-f071b91c6c0a kind: Event lastTimestamp: "2026-05-25T13:36:51Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-766ab-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-25T13:36:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:36:51Z" name: success-200-isvc-766ab-predictor-69d568958b-r85bq.18b2d2770f3e4850 namespace: kserve-ci-e2e-test resourceVersion: "25670" uid: 2db6f2fd-acd1-460c-971f-e914349f0ed2 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:52Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-766ab-predictor-69d568958b-r85bq namespace: kserve-ci-e2e-test resourceVersion: "25639" uid: 62f285ff-1f15-4a53-992e-f071b91c6c0a kind: Event lastTimestamp: "2026-05-25T13:36:52Z" message: Add eth0 [10.134.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T13:36:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T13:36:52Z" name: success-200-isvc-766ab-predictor-69d568958b-r85bq.18b2d2773f8c5476 namespace: kserve-ci-e2e-test resourceVersion: "25683" uid: 8657f072-ba69-4937-882a-2500a344a41e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-766ab-predictor-69d568958b-r85bq namespace: kserve-ci-e2e-test resourceVersion: "25632" uid: 62f285ff-1f15-4a53-992e-f071b91c6c0a kind: Event lastTimestamp: "2026-05-25T13:36:52Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-25T13:36:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:36:52Z" name: success-200-isvc-766ab-predictor-69d568958b-r85bq.18b2d27740e895f1 namespace: kserve-ci-e2e-test resourceVersion: "25685" uid: d1d5941a-e035-4a4f-b538-e6d1600c2d3c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-766ab-predictor-69d568958b-r85bq namespace: kserve-ci-e2e-test resourceVersion: "25632" uid: 62f285ff-1f15-4a53-992e-f071b91c6c0a kind: Event lastTimestamp: "2026-05-25T13:36:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T13:36:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:36:52Z" name: success-200-isvc-766ab-predictor-69d568958b-r85bq.18b2d27743f5497e namespace: kserve-ci-e2e-test resourceVersion: "25686" uid: 4c2e953c-bea0-4cf3-bbc6-5ae5af509525 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-766ab-predictor-69d568958b-r85bq namespace: kserve-ci-e2e-test resourceVersion: "25632" uid: 62f285ff-1f15-4a53-992e-f071b91c6c0a kind: Event lastTimestamp: "2026-05-25T13:36:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T13:36:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:36:52Z" name: success-200-isvc-766ab-predictor-69d568958b-r85bq.18b2d27744705a1b namespace: kserve-ci-e2e-test resourceVersion: "25687" uid: 04658deb-5b63-4fd1-a494-928638a10012 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-766ab-predictor-69d568958b-r85bq namespace: kserve-ci-e2e-test resourceVersion: "25632" uid: 62f285ff-1f15-4a53-992e-f071b91c6c0a kind: Event lastTimestamp: "2026-05-25T13:36:52Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-25T13:36:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:36:52Z" name: success-200-isvc-766ab-predictor-69d568958b-r85bq.18b2d277447ce760 namespace: kserve-ci-e2e-test resourceVersion: "25688" uid: ece7686e-fc9a-4d9d-a263-708a201d6517 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-766ab-predictor-69d568958b-r85bq namespace: kserve-ci-e2e-test resourceVersion: "25632" uid: 62f285ff-1f15-4a53-992e-f071b91c6c0a kind: Event lastTimestamp: "2026-05-25T13:36:52Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T13:36:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:36:52Z" name: success-200-isvc-766ab-predictor-69d568958b-r85bq.18b2d27747e54f82 namespace: kserve-ci-e2e-test resourceVersion: "25692" uid: 0638ffd0-464f-4c45-a1ff-a178ab6be56e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-766ab-predictor-69d568958b-r85bq namespace: kserve-ci-e2e-test resourceVersion: "25632" uid: 62f285ff-1f15-4a53-992e-f071b91c6c0a kind: Event lastTimestamp: "2026-05-25T13:36:52Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T13:36:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:36:52Z" name: success-200-isvc-766ab-predictor-69d568958b-r85bq.18b2d277486fb5f0 namespace: kserve-ci-e2e-test resourceVersion: "25693" uid: 366a8e05-cd1f-475f-bdf0-726d2101b8b5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T13:36:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-766ab-predictor-69d568958b-r85bq namespace: kserve-ci-e2e-test resourceVersion: "25632" uid: 62f285ff-1f15-4a53-992e-f071b91c6c0a kind: Event lastTimestamp: "2026-05-25T13:37:29Z" message: 'Readiness probe failed: dial tcp 10.134.0.43:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T13:36:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:37:29Z" name: success-200-isvc-766ab-predictor-69d568958b-r85bq.18b2d2779eea6e1a namespace: kserve-ci-e2e-test resourceVersion: "26364" uid: 26974a5a-7e5c-49a3-8396-f1cb3836d12d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:46:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-766ab-predictor-69d568958b-r85bq namespace: kserve-ci-e2e-test resourceVersion: "25632" uid: 62f285ff-1f15-4a53-992e-f071b91c6c0a kind: Event lastTimestamp: "2026-05-25T13:46:06Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T13:46:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:46:06Z" name: success-200-isvc-766ab-predictor-69d568958b-r85bq.18b2d2f83651c301 namespace: kserve-ci-e2e-test resourceVersion: "30038" uid: d86e0545-2e9a-43be-80b9-c4066f46f218 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:46:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-766ab-predictor-69d568958b-r85bq namespace: kserve-ci-e2e-test resourceVersion: "25632" uid: 62f285ff-1f15-4a53-992e-f071b91c6c0a kind: Event lastTimestamp: "2026-05-25T13:46:06Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T13:46:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:46:06Z" name: success-200-isvc-766ab-predictor-69d568958b-r85bq.18b2d2f8365265ac namespace: kserve-ci-e2e-test resourceVersion: "30039" uid: 3486ca6a-e62c-44e6-b7c9-c2c3e6c109a2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-766ab-predictor-69d568958b namespace: kserve-ci-e2e-test resourceVersion: "25613" uid: e094ce57-abd4-4a7b-b236-a03c3dd46ee0 kind: Event lastTimestamp: "2026-05-25T13:36:51Z" message: 'Created pod: success-200-isvc-766ab-predictor-69d568958b-r85bq' metadata: creationTimestamp: "2026-05-25T13:36:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:36:51Z" name: success-200-isvc-766ab-predictor-69d568958b.18b2d27701ef9d45 namespace: kserve-ci-e2e-test resourceVersion: "25631" uid: 9c7db137-6b2a-4da0-8582-7d6839e1846d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-766ab-predictor namespace: kserve-ci-e2e-test resourceVersion: "25607" uid: 85e9ffab-5ad0-4183-8eec-d7f3c5d04d66 kind: Event lastTimestamp: "2026-05-25T13:36:51Z" message: Scaled up replica set success-200-isvc-766ab-predictor-69d568958b from 0 to 1 metadata: creationTimestamp: "2026-05-25T13:36:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:36:51Z" name: success-200-isvc-766ab-predictor.18b2d277002f9d3e namespace: kserve-ci-e2e-test resourceVersion: "25620" uid: c01e99f2-14a8-4f82-9f2d-4ae5d3425576 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T13:37:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-766ab-predictor namespace: kserve-ci-e2e-test resourceVersion: "25612" uid: 59ea0810-135b-43f8-9950-b12570858f4b kind: Event lastTimestamp: "2026-05-25T13:37:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T13:37:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:37:36Z" name: success-200-isvc-766ab-predictor.18b2d27a7f1e28a8 namespace: kserve-ci-e2e-test resourceVersion: "26410" uid: 989d4255-7759-4572-b0c1-3490f2067f8e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T13:37:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-766ab-predictor namespace: kserve-ci-e2e-test resourceVersion: "25612" uid: 59ea0810-135b-43f8-9950-b12570858f4b kind: Event lastTimestamp: "2026-05-25T13:37:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T13:37:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:37:36Z" name: success-200-isvc-766ab-predictor.18b2d27a7f844036 namespace: kserve-ci-e2e-test resourceVersion: "26411" uid: e80befba-5b7e-40d1-be61-b524a95d0b2d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T13:37:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-766ab-predictor namespace: kserve-ci-e2e-test resourceVersion: "26108" uid: 59ea0810-135b-43f8-9950-b12570858f4b kind: Event lastTimestamp: "2026-05-25T13:38:06Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T13:37:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:38:06Z" name: success-200-isvc-766ab-predictor.18b2d284fc6b32ca namespace: kserve-ci-e2e-test resourceVersion: "26688" uid: 9c798ffa-d920-49d2-8ea3-7014ffb44de3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T13:37:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-766ab-predictor namespace: kserve-ci-e2e-test resourceVersion: "26108" uid: 59ea0810-135b-43f8-9950-b12570858f4b kind: Event lastTimestamp: "2026-05-25T13:38:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T13:37:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:38:06Z" name: success-200-isvc-766ab-predictor.18b2d284fccccc83 namespace: kserve-ci-e2e-test resourceVersion: "26689" uid: 524b96da-7355-43f8-aa41-707cf4e2dbaa reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T13:36:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-766ab namespace: kserve-ci-e2e-test resourceVersion: "25584" uid: 724d66fe-5604-471e-9f81-d854a31b9016 kind: Event lastTimestamp: "2026-05-25T13:37:40Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T13:36:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:37:40Z" name: success-200-isvc-766ab.18b2d276fffc230b namespace: kserve-ci-e2e-test resourceVersion: "26445" uid: ab90dcbb-cb4a-416a-a888-b3a6ac943d9e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-766ab namespace: kserve-ci-e2e-test resourceVersion: "25584" uid: 724d66fe-5604-471e-9f81-d854a31b9016 kind: Event lastTimestamp: "2026-05-25T13:36:51Z" message: 'Failed to update status for InferenceService "success-200-isvc-766ab": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-766ab": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T13:36:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:36:51Z" name: success-200-isvc-766ab.18b2d27700ae3d2e namespace: kserve-ci-e2e-test resourceVersion: "25623" uid: e9039c26-943a-4d81-a5b6-3b926d9670ec reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:36:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-766ab namespace: kserve-ci-e2e-test resourceVersion: "25584" uid: 724d66fe-5604-471e-9f81-d854a31b9016 kind: Event lastTimestamp: "2026-05-25T13:36:51Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-766ab": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T13:36:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:36:51Z" name: success-200-isvc-766ab.18b2d27700ae9f5f namespace: kserve-ci-e2e-test resourceVersion: "25625" uid: a5552add-7077-4419-b7b9-78c631ca19c2 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:37:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-766ab namespace: kserve-ci-e2e-test resourceVersion: "26443" uid: 724d66fe-5604-471e-9f81-d854a31b9016 kind: Event lastTimestamp: "2026-05-25T13:37:40Z" message: InferenceService [success-200-isvc-766ab] is Ready metadata: creationTimestamp: "2026-05-25T13:37:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:37:40Z" name: success-200-isvc-766ab.18b2d28259dbdf60 namespace: kserve-ci-e2e-test resourceVersion: "26444" uid: 8d016d86-0007-4119-98d6-f34c8e38d586 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T13:14:30.650950Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-889c1-predictor-746ccd4c5b-pjzvp namespace: kserve-ci-e2e-test resourceVersion: "14008" uid: cbdc46e1-4ffb-4a9d-9eef-6615a72fe025 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-889c1-predictor-746ccd4c5b-pjzvp to ip-10-0-135-103.ec2.internal metadata: creationTimestamp: "2026-05-25T13:14:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T13:14:30Z" name: success-200-isvc-889c1-predictor-746ccd4c5b-pjzvp.18b2d13edaa1798b namespace: kserve-ci-e2e-test resourceVersion: "14015" uid: 2541b49e-d628-4b32-b324-16361f6a1fa4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c46b5f76-4gnkb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:31Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-889c1-predictor-746ccd4c5b-pjzvp namespace: kserve-ci-e2e-test resourceVersion: "14025" uid: cbdc46e1-4ffb-4a9d-9eef-6615a72fe025 kind: Event lastTimestamp: "2026-05-25T13:14:31Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T13:14:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T13:14:31Z" name: success-200-isvc-889c1-predictor-746ccd4c5b-pjzvp.18b2d13ef4ddecda namespace: kserve-ci-e2e-test resourceVersion: "14099" uid: 5df1b437-f1cc-4b41-bb9e-764d12df7f86 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-889c1-predictor-746ccd4c5b-pjzvp namespace: kserve-ci-e2e-test resourceVersion: "14011" uid: cbdc46e1-4ffb-4a9d-9eef-6615a72fe025 kind: Event lastTimestamp: "2026-05-25T13:14:31Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1532" metadata: creationTimestamp: "2026-05-25T13:14:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:14:31Z" name: success-200-isvc-889c1-predictor-746ccd4c5b-pjzvp.18b2d13ef6498afa namespace: kserve-ci-e2e-test resourceVersion: "14103" uid: 9c0f0e58-3699-442e-a262-cbd646fa100d reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-889c1-predictor-746ccd4c5b-pjzvp namespace: kserve-ci-e2e-test resourceVersion: "14011" uid: cbdc46e1-4ffb-4a9d-9eef-6615a72fe025 kind: Event lastTimestamp: "2026-05-25T13:14:43Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1532" in 12.11s (12.11s including waiting). Image size: 1335713395 bytes.' metadata: creationTimestamp: "2026-05-25T13:14:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:14:43Z" name: success-200-isvc-889c1-predictor-746ccd4c5b-pjzvp.18b2d141c81af30a namespace: kserve-ci-e2e-test resourceVersion: "14293" uid: 86d71d41-86d7-4325-b917-065600badcea reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-889c1-predictor-746ccd4c5b-pjzvp namespace: kserve-ci-e2e-test resourceVersion: "14011" uid: cbdc46e1-4ffb-4a9d-9eef-6615a72fe025 kind: Event lastTimestamp: "2026-05-25T13:14:43Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T13:14:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:14:43Z" name: success-200-isvc-889c1-predictor-746ccd4c5b-pjzvp.18b2d141cb5ecab1 namespace: kserve-ci-e2e-test resourceVersion: "14295" uid: 50ff47fa-f1ef-4150-9bba-8090be6b3037 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-889c1-predictor-746ccd4c5b-pjzvp namespace: kserve-ci-e2e-test resourceVersion: "14011" uid: cbdc46e1-4ffb-4a9d-9eef-6615a72fe025 kind: Event lastTimestamp: "2026-05-25T13:14:43Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T13:14:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:14:43Z" name: success-200-isvc-889c1-predictor-746ccd4c5b-pjzvp.18b2d141cbd6a7c5 namespace: kserve-ci-e2e-test resourceVersion: "14296" uid: 19aded2c-5d2a-4e25-b350-bffe48ea8d16 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-889c1-predictor-746ccd4c5b-pjzvp namespace: kserve-ci-e2e-test resourceVersion: "14011" uid: cbdc46e1-4ffb-4a9d-9eef-6615a72fe025 kind: Event lastTimestamp: "2026-05-25T13:14:43Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-05-25T13:14:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:14:43Z" name: success-200-isvc-889c1-predictor-746ccd4c5b-pjzvp.18b2d141cbe2746f namespace: kserve-ci-e2e-test resourceVersion: "14298" uid: b2a7ac51-6f85-4d89-926b-843bf3af7485 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-889c1-predictor-746ccd4c5b-pjzvp namespace: kserve-ci-e2e-test resourceVersion: "14011" uid: cbdc46e1-4ffb-4a9d-9eef-6615a72fe025 kind: Event lastTimestamp: "2026-05-25T13:14:46Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 3.031s (3.031s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-05-25T13:14:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:14:46Z" name: success-200-isvc-889c1-predictor-746ccd4c5b-pjzvp.18b2d142808d6a53 namespace: kserve-ci-e2e-test resourceVersion: "14336" uid: cb579ced-5ee9-41c3-af01-c281539619dd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-889c1-predictor-746ccd4c5b-pjzvp namespace: kserve-ci-e2e-test resourceVersion: "14011" uid: cbdc46e1-4ffb-4a9d-9eef-6615a72fe025 kind: Event lastTimestamp: "2026-05-25T13:14:46Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T13:14:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:14:46Z" name: success-200-isvc-889c1-predictor-746ccd4c5b-pjzvp.18b2d1428503fd55 namespace: kserve-ci-e2e-test resourceVersion: "14338" uid: 63efff23-89a9-40d8-ab7d-1df3efc3a9d7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-889c1-predictor-746ccd4c5b-pjzvp namespace: kserve-ci-e2e-test resourceVersion: "14011" uid: cbdc46e1-4ffb-4a9d-9eef-6615a72fe025 kind: Event lastTimestamp: "2026-05-25T13:14:46Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T13:14:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:14:46Z" name: success-200-isvc-889c1-predictor-746ccd4c5b-pjzvp.18b2d14285b0c4cd namespace: kserve-ci-e2e-test resourceVersion: "14339" uid: f19b289c-6cc0-4779-ae4c-67cd26d15a3e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-05-25T13:14:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-889c1-predictor-746ccd4c5b-pjzvp namespace: kserve-ci-e2e-test resourceVersion: "14011" uid: cbdc46e1-4ffb-4a9d-9eef-6615a72fe025 kind: Event lastTimestamp: "2026-05-25T13:16:03Z" message: 'Readiness probe failed: dial tcp 10.134.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T13:14:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:16:03Z" name: success-200-isvc-889c1-predictor-746ccd4c5b-pjzvp.18b2d142c18e1bc5 namespace: kserve-ci-e2e-test resourceVersion: "15205" uid: 34634639-9179-4d5b-bcb5-4da391cf2552 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-889c1-predictor-746ccd4c5b-pjzvp namespace: kserve-ci-e2e-test resourceVersion: "14011" uid: cbdc46e1-4ffb-4a9d-9eef-6615a72fe025 kind: Event lastTimestamp: "2026-05-25T13:16:00Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T13:16:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:16:00Z" name: success-200-isvc-889c1-predictor-746ccd4c5b-pjzvp.18b2d153e1653880 namespace: kserve-ci-e2e-test resourceVersion: "15103" uid: 68b3f7c8-07b3-4148-be10-62c500942d7e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-889c1-predictor-746ccd4c5b-pjzvp namespace: kserve-ci-e2e-test resourceVersion: "14011" uid: cbdc46e1-4ffb-4a9d-9eef-6615a72fe025 kind: Event lastTimestamp: "2026-05-25T13:16:00Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T13:16:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:16:00Z" name: success-200-isvc-889c1-predictor-746ccd4c5b-pjzvp.18b2d153e165baec namespace: kserve-ci-e2e-test resourceVersion: "15106" uid: d8940378-f0ea-432f-b9cc-280b30d3d5a8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-889c1-predictor-746ccd4c5b-pjzvp namespace: kserve-ci-e2e-test resourceVersion: "14011" uid: cbdc46e1-4ffb-4a9d-9eef-6615a72fe025 kind: Event lastTimestamp: "2026-05-25T13:16:03Z" message: 'Readiness probe failed: Get "https://10.134.0.32:8643/healthz": dial tcp 10.134.0.32:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-25T13:16:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:16:03Z" name: success-200-isvc-889c1-predictor-746ccd4c5b-pjzvp.18b2d15473ba7e0d namespace: kserve-ci-e2e-test resourceVersion: "15204" uid: 375ab8a9-8467-43e7-ab58-0ffa719234c1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-889c1-predictor-746ccd4c5b namespace: kserve-ci-e2e-test resourceVersion: "14002" uid: 47c5bdb7-78b8-4bdd-8547-e5efbf9d18d3 kind: Event lastTimestamp: "2026-05-25T13:14:30Z" message: 'Created pod: success-200-isvc-889c1-predictor-746ccd4c5b-pjzvp' metadata: creationTimestamp: "2026-05-25T13:14:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:14:30Z" name: success-200-isvc-889c1-predictor-746ccd4c5b.18b2d13eda38a413 namespace: kserve-ci-e2e-test resourceVersion: "14013" uid: 0e20376f-72d3-4afa-ac2f-5be02c0daaae reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-889c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13998" uid: 0b18dc2f-0dff-4e6e-af0c-e90d73de5ea3 kind: Event lastTimestamp: "2026-05-25T13:14:30Z" message: Scaled up replica set success-200-isvc-889c1-predictor-746ccd4c5b from 0 to 1 metadata: creationTimestamp: "2026-05-25T13:14:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:14:30Z" name: success-200-isvc-889c1-predictor.18b2d13ed43ab3bf namespace: kserve-ci-e2e-test resourceVersion: "14004" uid: 9e5faca9-2260-4852-a87c-01d8cd37d814 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T13:14:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-889c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14009" uid: 0865f2ef-791e-4541-8504-896ea0c5ac7a kind: Event lastTimestamp: "2026-05-25T13:15:00Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T13:14:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:15:00Z" name: success-200-isvc-889c1-predictor.18b2d14259b21c1d namespace: kserve-ci-e2e-test resourceVersion: "14489" uid: 7541d967-dc0e-4ce6-8b90-e96ff8df1a3f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T13:14:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-889c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14009" uid: 0865f2ef-791e-4541-8504-896ea0c5ac7a kind: Event lastTimestamp: "2026-05-25T13:15:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T13:14:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:15:00Z" name: success-200-isvc-889c1-predictor.18b2d1425a22631a namespace: kserve-ci-e2e-test resourceVersion: "14490" uid: f16fa234-cabf-4445-9098-dd760c15d789 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-25T13:15:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-889c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14321" uid: 0865f2ef-791e-4541-8504-896ea0c5ac7a kind: Event lastTimestamp: "2026-05-25T13:16:00Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T13:15:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:16:00Z" name: success-200-isvc-889c1-predictor.18b2d14957eca503 namespace: kserve-ci-e2e-test resourceVersion: "15044" uid: c4ff603f-5cd9-4a3c-b8a5-c904429af831 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-25T13:15:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-889c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14321" uid: 0865f2ef-791e-4541-8504-896ea0c5ac7a kind: Event lastTimestamp: "2026-05-25T13:16:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T13:15:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:16:00Z" name: success-200-isvc-889c1-predictor.18b2d1495850f7ec namespace: kserve-ci-e2e-test resourceVersion: "15046" uid: b2f467f2-251f-437f-8ea2-78467a0e756a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T13:14:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-889c1 namespace: kserve-ci-e2e-test resourceVersion: "13990" uid: f851d38d-bb23-49e7-bf08-2f5be07e0954 kind: Event lastTimestamp: "2026-05-25T13:15:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T13:14:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:15:33Z" name: success-200-isvc-889c1.18b2d13eda7ab9e6 namespace: kserve-ci-e2e-test resourceVersion: "14769" uid: c89b203a-c131-493e-ad35-062043e0fe21 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-889c1 namespace: kserve-ci-e2e-test resourceVersion: "13990" uid: f851d38d-bb23-49e7-bf08-2f5be07e0954 kind: Event lastTimestamp: "2026-05-25T13:14:30Z" message: 'Failed to update status for InferenceService "success-200-isvc-889c1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-889c1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T13:14:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:14:30Z" name: success-200-isvc-889c1.18b2d13edae46940 namespace: kserve-ci-e2e-test resourceVersion: "14017" uid: 74d7f980-6902-4b90-a23c-5f75e7e03b38 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:14:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-889c1 namespace: kserve-ci-e2e-test resourceVersion: "13990" uid: f851d38d-bb23-49e7-bf08-2f5be07e0954 kind: Event lastTimestamp: "2026-05-25T13:14:30Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-889c1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T13:14:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:14:30Z" name: success-200-isvc-889c1.18b2d13edae4ca28 namespace: kserve-ci-e2e-test resourceVersion: "14020" uid: 3b6cce60-6678-4631-a02d-fc926082e8a1 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:15:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-889c1 namespace: kserve-ci-e2e-test resourceVersion: "14766" uid: f851d38d-bb23-49e7-bf08-2f5be07e0954 kind: Event lastTimestamp: "2026-05-25T13:15:33Z" message: InferenceService [success-200-isvc-889c1] is Ready metadata: creationTimestamp: "2026-05-25T13:15:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:15:33Z" name: success-200-isvc-889c1.18b2d14d7c029828 namespace: kserve-ci-e2e-test resourceVersion: "14767" uid: fb52e0ad-920b-4adc-874a-5a5b8a3a59f0 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T13:25:45.906341Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-9ee57-predictor-767dfdd744-jlp7p namespace: kserve-ci-e2e-test resourceVersion: "19847" uid: 504712a1-9a41-4d39-a504-32f340cfa820 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-9ee57-predictor-767dfdd744-jlp7p to ip-10-0-135-103.ec2.internal metadata: creationTimestamp: "2026-05-25T13:25:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T13:25:45Z" name: success-200-isvc-9ee57-predictor-767dfdd744-jlp7p.18b2d1dc12fd0477 namespace: kserve-ci-e2e-test resourceVersion: "19854" uid: 203f6481-06d6-48b5-a949-2b8d0f6ea296 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c46b5f76-4gnkb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:25:46Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-9ee57-predictor-767dfdd744-jlp7p namespace: kserve-ci-e2e-test resourceVersion: "19867" uid: 504712a1-9a41-4d39-a504-32f340cfa820 kind: Event lastTimestamp: "2026-05-25T13:25:46Z" message: Add eth0 [10.134.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T13:25:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T13:25:46Z" name: success-200-isvc-9ee57-predictor-767dfdd744-jlp7p.18b2d1dc2c93aaeb namespace: kserve-ci-e2e-test resourceVersion: "19898" uid: 7e18cf80-31d3-4b11-9218-d98d26b5c665 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:25:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9ee57-predictor-767dfdd744-jlp7p namespace: kserve-ci-e2e-test resourceVersion: "19851" uid: 504712a1-9a41-4d39-a504-32f340cfa820 kind: Event lastTimestamp: "2026-05-25T13:25:46Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-25T13:25:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:25:46Z" name: success-200-isvc-9ee57-predictor-767dfdd744-jlp7p.18b2d1dc2df1ec51 namespace: kserve-ci-e2e-test resourceVersion: "19903" uid: f76e5a21-bc45-4ff5-b37f-e88ee0f9f080 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:25:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9ee57-predictor-767dfdd744-jlp7p namespace: kserve-ci-e2e-test resourceVersion: "19851" uid: 504712a1-9a41-4d39-a504-32f340cfa820 kind: Event lastTimestamp: "2026-05-25T13:25:46Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T13:25:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:25:46Z" name: success-200-isvc-9ee57-predictor-767dfdd744-jlp7p.18b2d1dc313a900c namespace: kserve-ci-e2e-test resourceVersion: "19904" uid: 4852d9cd-cea0-4eb1-a177-5f58a8575255 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:25:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9ee57-predictor-767dfdd744-jlp7p namespace: kserve-ci-e2e-test resourceVersion: "19851" uid: 504712a1-9a41-4d39-a504-32f340cfa820 kind: Event lastTimestamp: "2026-05-25T13:25:46Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T13:25:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:25:46Z" name: success-200-isvc-9ee57-predictor-767dfdd744-jlp7p.18b2d1dc31afc0ae namespace: kserve-ci-e2e-test resourceVersion: "19905" uid: a6d49cdb-581b-4616-a895-4e8b44a2a33e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:25:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9ee57-predictor-767dfdd744-jlp7p namespace: kserve-ci-e2e-test resourceVersion: "19851" uid: 504712a1-9a41-4d39-a504-32f340cfa820 kind: Event lastTimestamp: "2026-05-25T13:25:46Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-25T13:25:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:25:46Z" name: success-200-isvc-9ee57-predictor-767dfdd744-jlp7p.18b2d1dc31bac81c namespace: kserve-ci-e2e-test resourceVersion: "19906" uid: 61c2403e-153c-433e-9364-0fc996f69111 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:25:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9ee57-predictor-767dfdd744-jlp7p namespace: kserve-ci-e2e-test resourceVersion: "19851" uid: 504712a1-9a41-4d39-a504-32f340cfa820 kind: Event lastTimestamp: "2026-05-25T13:25:46Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T13:25:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:25:46Z" name: success-200-isvc-9ee57-predictor-767dfdd744-jlp7p.18b2d1dc35d85611 namespace: kserve-ci-e2e-test resourceVersion: "19907" uid: 98318a43-2490-4b4d-8e66-ee80df965559 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:25:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9ee57-predictor-767dfdd744-jlp7p namespace: kserve-ci-e2e-test resourceVersion: "19851" uid: 504712a1-9a41-4d39-a504-32f340cfa820 kind: Event lastTimestamp: "2026-05-25T13:25:46Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T13:25:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:25:46Z" name: success-200-isvc-9ee57-predictor-767dfdd744-jlp7p.18b2d1dc3657a5f7 namespace: kserve-ci-e2e-test resourceVersion: "19908" uid: 8285f3ec-d3cb-4475-8c52-0e85a971e466 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T13:25:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9ee57-predictor-767dfdd744-jlp7p namespace: kserve-ci-e2e-test resourceVersion: "19851" uid: 504712a1-9a41-4d39-a504-32f340cfa820 kind: Event lastTimestamp: "2026-05-25T13:26:23Z" message: 'Readiness probe failed: dial tcp 10.134.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T13:25:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:26:23Z" name: success-200-isvc-9ee57-predictor-767dfdd744-jlp7p.18b2d1dc7dc177f2 namespace: kserve-ci-e2e-test resourceVersion: "20290" uid: 1b66c549-7c1d-422c-898a-43aab3542f6c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9ee57-predictor-767dfdd744-jlp7p namespace: kserve-ci-e2e-test resourceVersion: "19851" uid: 504712a1-9a41-4d39-a504-32f340cfa820 kind: Event lastTimestamp: "2026-05-25T13:26:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T13:26:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:26:56Z" name: success-200-isvc-9ee57-predictor-767dfdd744-jlp7p.18b2d1ec6cf82328 namespace: kserve-ci-e2e-test resourceVersion: "20795" uid: 26d63ebd-c8aa-4697-86c2-5686bf422595 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9ee57-predictor-767dfdd744-jlp7p namespace: kserve-ci-e2e-test resourceVersion: "19851" uid: 504712a1-9a41-4d39-a504-32f340cfa820 kind: Event lastTimestamp: "2026-05-25T13:26:56Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T13:26:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:26:56Z" name: success-200-isvc-9ee57-predictor-767dfdd744-jlp7p.18b2d1ec6cfc0194 namespace: kserve-ci-e2e-test resourceVersion: "20803" uid: 8e5a0e69-73a2-4698-a32e-895fafe209a5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9ee57-predictor-767dfdd744-jlp7p namespace: kserve-ci-e2e-test resourceVersion: "19851" uid: 504712a1-9a41-4d39-a504-32f340cfa820 kind: Event lastTimestamp: "2026-05-25T13:26:58Z" message: 'Readiness probe failed: Get "https://10.134.0.38:8643/healthz": dial tcp 10.134.0.38:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-25T13:26:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:26:58Z" name: success-200-isvc-9ee57-predictor-767dfdd744-jlp7p.18b2d1ed05d52b86 namespace: kserve-ci-e2e-test resourceVersion: "20908" uid: 25ee4d34-1aa5-4e23-92b8-29b33c7e15ac reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:25:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-9ee57-predictor-767dfdd744 namespace: kserve-ci-e2e-test resourceVersion: "19830" uid: 35949f7b-964d-4cdb-a655-43dae8baa37d kind: Event lastTimestamp: "2026-05-25T13:25:45Z" message: 'Created pod: success-200-isvc-9ee57-predictor-767dfdd744-jlp7p' metadata: creationTimestamp: "2026-05-25T13:25:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:25:45Z" name: success-200-isvc-9ee57-predictor-767dfdd744.18b2d1dc126b5c6b namespace: kserve-ci-e2e-test resourceVersion: "19850" uid: ce9486e9-0bb3-4355-8e8e-fc1fb3cc9cb2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:25:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-9ee57-predictor namespace: kserve-ci-e2e-test resourceVersion: "19827" uid: 9effcad8-1f58-4760-9588-a0e9933b7105 kind: Event lastTimestamp: "2026-05-25T13:25:45Z" message: Scaled up replica set success-200-isvc-9ee57-predictor-767dfdd744 from 0 to 1 metadata: creationTimestamp: "2026-05-25T13:25:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:25:45Z" name: success-200-isvc-9ee57-predictor.18b2d1dc10a43e47 namespace: kserve-ci-e2e-test resourceVersion: "19833" uid: 95b7819d-0cd5-47ff-bdda-caefd722b74d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T13:26:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9ee57-predictor namespace: kserve-ci-e2e-test resourceVersion: "19828" uid: 694998dd-5725-47ab-a69f-c2e4e4419305 kind: Event lastTimestamp: "2026-05-25T13:26:15Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T13:26:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:26:15Z" name: success-200-isvc-9ee57-predictor.18b2d1df8f27f12c namespace: kserve-ci-e2e-test resourceVersion: "20193" uid: 44b6b49c-1651-48d8-9cb4-1681e097037e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T13:26:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9ee57-predictor namespace: kserve-ci-e2e-test resourceVersion: "19828" uid: 694998dd-5725-47ab-a69f-c2e4e4419305 kind: Event lastTimestamp: "2026-05-25T13:26:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T13:26:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:26:15Z" name: success-200-isvc-9ee57-predictor.18b2d1df8f6fb71e namespace: kserve-ci-e2e-test resourceVersion: "20197" uid: 2d6a6fca-f02d-4c0b-adef-bbb9f3535369 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T13:26:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9ee57-predictor namespace: kserve-ci-e2e-test resourceVersion: "20048" uid: 694998dd-5725-47ab-a69f-c2e4e4419305 kind: Event lastTimestamp: "2026-05-25T13:26:45Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T13:26:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:26:45Z" name: success-200-isvc-9ee57-predictor.18b2d1e68ce3b016 namespace: kserve-ci-e2e-test resourceVersion: "20649" uid: 30d268e2-ee59-4291-aeb3-10475d1dc802 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T13:26:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9ee57-predictor namespace: kserve-ci-e2e-test resourceVersion: "20048" uid: 694998dd-5725-47ab-a69f-c2e4e4419305 kind: Event lastTimestamp: "2026-05-25T13:26:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T13:26:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:26:45Z" name: success-200-isvc-9ee57-predictor.18b2d1e68d4f02a3 namespace: kserve-ci-e2e-test resourceVersion: "20651" uid: 674379c4-dcbd-4291-b873-44c3f4492afb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T13:25:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9ee57 namespace: kserve-ci-e2e-test resourceVersion: "19802" uid: ba1b3f49-36f9-4691-9bb5-bb26767c4cf1 kind: Event lastTimestamp: "2026-05-25T13:26:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T13:25:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:26:33Z" name: success-200-isvc-9ee57.18b2d1dc1076eba8 namespace: kserve-ci-e2e-test resourceVersion: "20521" uid: faf36071-ceb2-40ca-92fc-404d6d5ca291 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:25:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9ee57 namespace: kserve-ci-e2e-test resourceVersion: "19802" uid: ba1b3f49-36f9-4691-9bb5-bb26767c4cf1 kind: Event lastTimestamp: "2026-05-25T13:25:45Z" message: 'Failed to update status for InferenceService "success-200-isvc-9ee57": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-9ee57": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T13:25:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:25:45Z" name: success-200-isvc-9ee57.18b2d1dc10d0f768 namespace: kserve-ci-e2e-test resourceVersion: "19835" uid: 3b2e74d4-bb96-4e3c-bd96-0b0a9fa50904 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:25:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9ee57 namespace: kserve-ci-e2e-test resourceVersion: "19802" uid: ba1b3f49-36f9-4691-9bb5-bb26767c4cf1 kind: Event lastTimestamp: "2026-05-25T13:25:45Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-9ee57": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T13:25:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:25:45Z" name: success-200-isvc-9ee57.18b2d1dc10d1682b namespace: kserve-ci-e2e-test resourceVersion: "19839" uid: c4d59b15-23ef-4f22-b64d-09d3eb94024b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9ee57 namespace: kserve-ci-e2e-test resourceVersion: "20519" uid: ba1b3f49-36f9-4691-9bb5-bb26767c4cf1 kind: Event lastTimestamp: "2026-05-25T13:26:33Z" message: InferenceService [success-200-isvc-9ee57] is Ready metadata: creationTimestamp: "2026-05-25T13:26:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:26:33Z" name: success-200-isvc-9ee57.18b2d1e7388f8321 namespace: kserve-ci-e2e-test resourceVersion: "20520" uid: 1094f225-1083-4230-b6bf-3d00c5ddba55 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T13:25:15.842021Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-db114-predictor-586c7dfd-52v5d namespace: kserve-ci-e2e-test resourceVersion: "19480" uid: 830a1fff-253d-45b7-8c19-b896f15c0bc9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-db114-predictor-586c7dfd-52v5d to ip-10-0-135-103.ec2.internal metadata: creationTimestamp: "2026-05-25T13:25:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T13:25:15Z" name: success-200-isvc-db114-predictor-586c7dfd-52v5d.18b2d1d51303efff namespace: kserve-ci-e2e-test resourceVersion: "19486" uid: dcb3cd0b-6718-41f8-86dc-b3ab0f3c9126 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c46b5f76-4gnkb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:25:16Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-db114-predictor-586c7dfd-52v5d namespace: kserve-ci-e2e-test resourceVersion: "19498" uid: 830a1fff-253d-45b7-8c19-b896f15c0bc9 kind: Event lastTimestamp: "2026-05-25T13:25:16Z" message: Add eth0 [10.134.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T13:25:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T13:25:16Z" name: success-200-isvc-db114-predictor-586c7dfd-52v5d.18b2d1d52c6fa4f3 namespace: kserve-ci-e2e-test resourceVersion: "19526" uid: 8753ad35-a54b-43fd-8304-e179fe659968 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:25:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-db114-predictor-586c7dfd-52v5d namespace: kserve-ci-e2e-test resourceVersion: "19483" uid: 830a1fff-253d-45b7-8c19-b896f15c0bc9 kind: Event lastTimestamp: "2026-05-25T13:25:16Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-25T13:25:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:25:16Z" name: success-200-isvc-db114-predictor-586c7dfd-52v5d.18b2d1d52dea0236 namespace: kserve-ci-e2e-test resourceVersion: "19528" uid: d2026e5e-16e7-455b-bdcf-2083fcaa61c9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:25:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-db114-predictor-586c7dfd-52v5d namespace: kserve-ci-e2e-test resourceVersion: "19483" uid: 830a1fff-253d-45b7-8c19-b896f15c0bc9 kind: Event lastTimestamp: "2026-05-25T13:25:16Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T13:25:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:25:16Z" name: success-200-isvc-db114-predictor-586c7dfd-52v5d.18b2d1d530e524d3 namespace: kserve-ci-e2e-test resourceVersion: "19530" uid: f078f216-126a-487e-b586-70bc03d832a4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:25:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-db114-predictor-586c7dfd-52v5d namespace: kserve-ci-e2e-test resourceVersion: "19483" uid: 830a1fff-253d-45b7-8c19-b896f15c0bc9 kind: Event lastTimestamp: "2026-05-25T13:25:16Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T13:25:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:25:16Z" name: success-200-isvc-db114-predictor-586c7dfd-52v5d.18b2d1d5318957b1 namespace: kserve-ci-e2e-test resourceVersion: "19532" uid: 860eb0bf-21e8-4fc4-9d7d-5006f002b5ea reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:25:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-db114-predictor-586c7dfd-52v5d namespace: kserve-ci-e2e-test resourceVersion: "19483" uid: 830a1fff-253d-45b7-8c19-b896f15c0bc9 kind: Event lastTimestamp: "2026-05-25T13:25:16Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-25T13:25:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:25:16Z" name: success-200-isvc-db114-predictor-586c7dfd-52v5d.18b2d1d53195aa99 namespace: kserve-ci-e2e-test resourceVersion: "19534" uid: a7f69c66-9f7f-4829-bc58-fb8ee71d9a38 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:25:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-db114-predictor-586c7dfd-52v5d namespace: kserve-ci-e2e-test resourceVersion: "19483" uid: 830a1fff-253d-45b7-8c19-b896f15c0bc9 kind: Event lastTimestamp: "2026-05-25T13:25:16Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T13:25:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:25:16Z" name: success-200-isvc-db114-predictor-586c7dfd-52v5d.18b2d1d534bee40f namespace: kserve-ci-e2e-test resourceVersion: "19535" uid: e65563c1-b76f-4f03-a108-dcef4ed5be95 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:25:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-db114-predictor-586c7dfd-52v5d namespace: kserve-ci-e2e-test resourceVersion: "19483" uid: 830a1fff-253d-45b7-8c19-b896f15c0bc9 kind: Event lastTimestamp: "2026-05-25T13:25:16Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T13:25:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:25:16Z" name: success-200-isvc-db114-predictor-586c7dfd-52v5d.18b2d1d53536d7d9 namespace: kserve-ci-e2e-test resourceVersion: "19536" uid: 840ab535-4c0e-41fd-b640-7d4334b13f1d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T13:25:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-db114-predictor-586c7dfd-52v5d namespace: kserve-ci-e2e-test resourceVersion: "19483" uid: 830a1fff-253d-45b7-8c19-b896f15c0bc9 kind: Event lastTimestamp: "2026-05-25T13:25:52Z" message: 'Readiness probe failed: dial tcp 10.134.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T13:25:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:25:52Z" name: success-200-isvc-db114-predictor-586c7dfd-52v5d.18b2d1d53ef917bc namespace: kserve-ci-e2e-test resourceVersion: "19983" uid: 7a184c35-a0df-4d5e-ad1b-b1e62791cced reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-db114-predictor-586c7dfd-52v5d namespace: kserve-ci-e2e-test resourceVersion: "19483" uid: 830a1fff-253d-45b7-8c19-b896f15c0bc9 kind: Event lastTimestamp: "2026-05-25T13:26:26Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T13:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:26:26Z" name: success-200-isvc-db114-predictor-586c7dfd-52v5d.18b2d1e5715c0d61 namespace: kserve-ci-e2e-test resourceVersion: "20353" uid: fd4ae7e3-deda-4dcd-97ce-7082c65b0053 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-db114-predictor-586c7dfd-52v5d namespace: kserve-ci-e2e-test resourceVersion: "19483" uid: 830a1fff-253d-45b7-8c19-b896f15c0bc9 kind: Event lastTimestamp: "2026-05-25T13:26:26Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T13:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:26:26Z" name: success-200-isvc-db114-predictor-586c7dfd-52v5d.18b2d1e5715d1f39 namespace: kserve-ci-e2e-test resourceVersion: "20355" uid: fa9830e6-cae6-48b6-9295-b6bf4dfb4753 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-db114-predictor-586c7dfd-52v5d namespace: kserve-ci-e2e-test resourceVersion: "19483" uid: 830a1fff-253d-45b7-8c19-b896f15c0bc9 kind: Event lastTimestamp: "2026-05-25T13:26:27Z" message: 'Readiness probe failed: Get "https://10.134.0.37:8643/healthz": dial tcp 10.134.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-25T13:26:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:26:27Z" name: success-200-isvc-db114-predictor-586c7dfd-52v5d.18b2d1e5c71f3d82 namespace: kserve-ci-e2e-test resourceVersion: "20450" uid: d1d5f53f-ad99-4e77-a298-e8b97b0a2bff reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-103.ec2.internal source: component: kubelet host: ip-10-0-135-103.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:25:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-db114-predictor-586c7dfd namespace: kserve-ci-e2e-test resourceVersion: "19460" uid: 03925a11-0ddb-4200-8bd3-4985427c43f7 kind: Event lastTimestamp: "2026-05-25T13:25:15Z" message: 'Created pod: success-200-isvc-db114-predictor-586c7dfd-52v5d' metadata: creationTimestamp: "2026-05-25T13:25:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:25:15Z" name: success-200-isvc-db114-predictor-586c7dfd.18b2d1d5129080f1 namespace: kserve-ci-e2e-test resourceVersion: "19482" uid: 3b2918c9-21fe-4c19-a47e-77e6bc75847f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:25:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-db114-predictor namespace: kserve-ci-e2e-test resourceVersion: "19454" uid: c63e94e4-939b-48bd-aa75-0236283fb23a kind: Event lastTimestamp: "2026-05-25T13:25:15Z" message: Scaled up replica set success-200-isvc-db114-predictor-586c7dfd from 0 to 1 metadata: creationTimestamp: "2026-05-25T13:25:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:25:15Z" name: success-200-isvc-db114-predictor.18b2d1d50f080860 namespace: kserve-ci-e2e-test resourceVersion: "19465" uid: cd4c1543-a3d6-4a9f-8c65-8f044c173f2f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T13:25:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-db114-predictor namespace: kserve-ci-e2e-test resourceVersion: "19459" uid: 6caa7f6e-cdd2-4a99-b799-b70ed860acd6 kind: Event lastTimestamp: "2026-05-25T13:25:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T13:25:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:25:45Z" name: success-200-isvc-db114-predictor.18b2d1d88da3f71c namespace: kserve-ci-e2e-test resourceVersion: "19804" uid: 1529d516-558e-4de2-9f66-80dc09539a67 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T13:25:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-db114-predictor namespace: kserve-ci-e2e-test resourceVersion: "19459" uid: 6caa7f6e-cdd2-4a99-b799-b70ed860acd6 kind: Event lastTimestamp: "2026-05-25T13:25:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T13:25:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:25:45Z" name: success-200-isvc-db114-predictor.18b2d1d88e04d31a namespace: kserve-ci-e2e-test resourceVersion: "19809" uid: 1c63a710-a6ff-401c-9e07-7f2fd15a6d28 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T13:26:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-db114-predictor namespace: kserve-ci-e2e-test resourceVersion: "19667" uid: 6caa7f6e-cdd2-4a99-b799-b70ed860acd6 kind: Event lastTimestamp: "2026-05-25T13:26:15Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T13:26:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:26:15Z" name: success-200-isvc-db114-predictor.18b2d1df8b71fe14 namespace: kserve-ci-e2e-test resourceVersion: "20185" uid: 251e8082-b15d-4651-8b2c-e17c6f851ab8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T13:26:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-db114-predictor namespace: kserve-ci-e2e-test resourceVersion: "19667" uid: 6caa7f6e-cdd2-4a99-b799-b70ed860acd6 kind: Event lastTimestamp: "2026-05-25T13:26:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T13:26:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:26:15Z" name: success-200-isvc-db114-predictor.18b2d1df8bbd4141 namespace: kserve-ci-e2e-test resourceVersion: "20186" uid: c61d2a9a-9ead-41f7-a64c-298efe5f34c3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T13:25:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-db114 namespace: kserve-ci-e2e-test resourceVersion: "19431" uid: 080f3904-92e1-43f6-b4ed-fb9f87b2d6eb kind: Event lastTimestamp: "2026-05-25T13:26:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T13:25:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:26:02Z" name: success-200-isvc-db114.18b2d1d50efedf5f namespace: kserve-ci-e2e-test resourceVersion: "20079" uid: 0337507d-2e15-4435-bd75-1d8794115667 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:25:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-db114 namespace: kserve-ci-e2e-test resourceVersion: "19431" uid: 080f3904-92e1-43f6-b4ed-fb9f87b2d6eb kind: Event lastTimestamp: "2026-05-25T13:25:15Z" message: 'Failed to update status for InferenceService "success-200-isvc-db114": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-db114": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T13:25:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:25:15Z" name: success-200-isvc-db114.18b2d1d50f96ccfd namespace: kserve-ci-e2e-test resourceVersion: "19471" uid: ce9703b9-0e13-475a-a1cc-82886958bdd2 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:25:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-db114 namespace: kserve-ci-e2e-test resourceVersion: "19431" uid: 080f3904-92e1-43f6-b4ed-fb9f87b2d6eb kind: Event lastTimestamp: "2026-05-25T13:25:15Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-db114": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T13:25:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:25:15Z" name: success-200-isvc-db114.18b2d1d50f972179 namespace: kserve-ci-e2e-test resourceVersion: "19473" uid: 97e89333-28c7-4e7b-be05-9a0244de0b90 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:26:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-db114 namespace: kserve-ci-e2e-test resourceVersion: "20076" uid: 080f3904-92e1-43f6-b4ed-fb9f87b2d6eb kind: Event lastTimestamp: "2026-05-25T13:26:02Z" message: InferenceService [success-200-isvc-db114] is Ready metadata: creationTimestamp: "2026-05-25T13:26:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:26:02Z" name: success-200-isvc-db114.18b2d1dff9f25c72 namespace: kserve-ci-e2e-test resourceVersion: "20078" uid: 85716468-2bf4-4a75-9fa7-c763579e9731 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T13:37:11.156689Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-12313-76b55b9df6-96qlq namespace: kserve-ci-e2e-test resourceVersion: "26187" uid: 915f615d-8e11-4584-bb10-57733fba7e83 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-12313-76b55b9df6-96qlq to ip-10-0-139-18.ec2.internal metadata: creationTimestamp: "2026-05-25T13:37:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T13:37:11Z" name: switch-graph-12313-76b55b9df6-96qlq.18b2d27b9f179366 namespace: kserve-ci-e2e-test resourceVersion: "26192" uid: d239687a-d586-4e03-8e67-09c4f3c03127 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c46b5f76-4gnkb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:37:11Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-12313-76b55b9df6-96qlq namespace: kserve-ci-e2e-test resourceVersion: "26195" uid: 915f615d-8e11-4584-bb10-57733fba7e83 kind: Event lastTimestamp: "2026-05-25T13:37:11Z" message: Add eth0 [10.132.0.46/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T13:37:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T13:37:11Z" name: switch-graph-12313-76b55b9df6-96qlq.18b2d27bb8e86ab9 namespace: kserve-ci-e2e-test resourceVersion: "26208" uid: b9426e39-e636-47cc-9470-4d5c3319271e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:37:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-12313} kind: Pod name: switch-graph-12313-76b55b9df6-96qlq namespace: kserve-ci-e2e-test resourceVersion: "26188" uid: 915f615d-8e11-4584-bb10-57733fba7e83 kind: Event lastTimestamp: "2026-05-25T13:37:11Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:367463b1d3262f4416182663b2344c09f7ccedc3764563fd3d0aee32ec772aae" already present on machine metadata: creationTimestamp: "2026-05-25T13:37:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:37:11Z" name: switch-graph-12313-76b55b9df6-96qlq.18b2d27bba5aa5d9 namespace: kserve-ci-e2e-test resourceVersion: "26210" uid: 56ea80c1-575f-4d7f-b6d1-e78d5d6c99d4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:37:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-12313} kind: Pod name: switch-graph-12313-76b55b9df6-96qlq namespace: kserve-ci-e2e-test resourceVersion: "26188" uid: 915f615d-8e11-4584-bb10-57733fba7e83 kind: Event lastTimestamp: "2026-05-25T13:37:11Z" message: 'Created container: switch-graph-12313' metadata: creationTimestamp: "2026-05-25T13:37:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:37:11Z" name: switch-graph-12313-76b55b9df6-96qlq.18b2d27bbdcdf8d9 namespace: kserve-ci-e2e-test resourceVersion: "26211" uid: 546344ba-3d57-480d-8c18-16c6407baa03 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:37:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-12313} kind: Pod name: switch-graph-12313-76b55b9df6-96qlq namespace: kserve-ci-e2e-test resourceVersion: "26188" uid: 915f615d-8e11-4584-bb10-57733fba7e83 kind: Event lastTimestamp: "2026-05-25T13:37:11Z" message: Started container switch-graph-12313 metadata: creationTimestamp: "2026-05-25T13:37:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:37:11Z" name: switch-graph-12313-76b55b9df6-96qlq.18b2d27bbe451eec namespace: kserve-ci-e2e-test resourceVersion: "26212" uid: b2b783d0-f41b-40f4-b021-5e79d2437943 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:53:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-12313} kind: Pod name: switch-graph-12313-76b55b9df6-96qlq namespace: kserve-ci-e2e-test resourceVersion: "26188" uid: 915f615d-8e11-4584-bb10-57733fba7e83 kind: Event lastTimestamp: "2026-05-25T13:53:30Z" message: Stopping container switch-graph-12313 metadata: creationTimestamp: "2026-05-25T13:53:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:53:30Z" name: switch-graph-12313-76b55b9df6-96qlq.18b2d35fa9040f94 namespace: kserve-ci-e2e-test resourceVersion: "33101" uid: a3486347-0a2e-4afe-81d3-909e69f2df37 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T13:53:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-12313} kind: Pod name: switch-graph-12313-76b55b9df6-96qlq namespace: kserve-ci-e2e-test resourceVersion: "26188" uid: 915f615d-8e11-4584-bb10-57733fba7e83 kind: Event lastTimestamp: "2026-05-25T13:53:53Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-25T13:53:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:53:53Z" name: switch-graph-12313-76b55b9df6-96qlq.18b2d36047d3d0b5 namespace: kserve-ci-e2e-test resourceVersion: "33337" uid: 0b4ad45f-195b-4ddf-aab8-19359b546f8b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:37:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-12313-76b55b9df6 namespace: kserve-ci-e2e-test resourceVersion: "26181" uid: 4a866f15-8365-4680-bd74-df9a807b68e9 kind: Event lastTimestamp: "2026-05-25T13:37:11Z" message: 'Created pod: switch-graph-12313-76b55b9df6-96qlq' metadata: creationTimestamp: "2026-05-25T13:37:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:37:11Z" name: switch-graph-12313-76b55b9df6.18b2d27b9e979410 namespace: kserve-ci-e2e-test resourceVersion: "26189" uid: 244c969a-f370-47d9-bdf2-56ff24b5f0fb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:37:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-12313 namespace: kserve-ci-e2e-test resourceVersion: "26179" uid: 1895d260-e288-4fb8-bc03-3ad44d45e5bc kind: Event lastTimestamp: "2026-05-25T13:37:11Z" message: Scaled up replica set switch-graph-12313-76b55b9df6 from 0 to 1 metadata: creationTimestamp: "2026-05-25T13:37:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:37:11Z" name: switch-graph-12313.18b2d27b9d6fc293 namespace: kserve-ci-e2e-test resourceVersion: "26183" uid: c5244229-a27c-4c3a-b3af-4f5039c0b88b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:37:11Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-12313 namespace: kserve-ci-e2e-test resourceVersion: "26198" uid: c8f00aae-72a6-46f6-b3fa-3d597952a09b kind: Event lastTimestamp: "2026-05-25T13:37:11Z" message: InferenceGraph [switch-graph-12313] is Ready metadata: creationTimestamp: "2026-05-25T13:37:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:37:11Z" name: switch-graph-12313.18b2d27ba3738290 namespace: kserve-ci-e2e-test resourceVersion: "26200" uid: 80c4f6b5-d7af-43ed-afb5-abc3259bcf7d reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T13:37:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-12313 namespace: kserve-ci-e2e-test resourceVersion: "26180" uid: 448a5bc0-4a59-43e2-a5b3-cc5b9973a3e7 kind: Event lastTimestamp: "2026-05-25T13:37:41Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T13:37:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:37:41Z" name: switch-graph-12313.18b2d27f1bf35da6 namespace: kserve-ci-e2e-test resourceVersion: "26460" uid: 3fccfa0f-b96e-4a09-9898-245fe02e2c1e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T13:37:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-12313 namespace: kserve-ci-e2e-test resourceVersion: "26180" uid: 448a5bc0-4a59-43e2-a5b3-cc5b9973a3e7 kind: Event lastTimestamp: "2026-05-25T13:37:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T13:37:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:37:41Z" name: switch-graph-12313.18b2d27f1c5809d8 namespace: kserve-ci-e2e-test resourceVersion: "26461" uid: a1044740-43c9-43c0-9d08-d1d8526d1731 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-05-25T13:17:01.078227Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-339a4-68f7b7cdb-6xr4s namespace: kserve-ci-e2e-test resourceVersion: "15936" uid: 192093cf-923b-4607-8d51-3f0edab2a14e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-339a4-68f7b7cdb-6xr4s to ip-10-0-139-18.ec2.internal metadata: creationTimestamp: "2026-05-25T13:17:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T13:17:01Z" name: switch-graph-339a4-68f7b7cdb-6xr4s.18b2d161e0cb7f09 namespace: kserve-ci-e2e-test resourceVersion: "15942" uid: 119d2bb2-7918-445f-9248-9d61f7a2af21 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c46b5f76-4gnkb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:17:01Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-339a4-68f7b7cdb-6xr4s namespace: kserve-ci-e2e-test resourceVersion: "15945" uid: 192093cf-923b-4607-8d51-3f0edab2a14e kind: Event lastTimestamp: "2026-05-25T13:17:01Z" message: Add eth0 [10.132.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T13:17:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T13:17:01Z" name: switch-graph-339a4-68f7b7cdb-6xr4s.18b2d161fa471f9e namespace: kserve-ci-e2e-test resourceVersion: "15963" uid: 0386ade2-bb34-4504-98f9-499c008589ca reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:17:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-339a4} kind: Pod name: switch-graph-339a4-68f7b7cdb-6xr4s namespace: kserve-ci-e2e-test resourceVersion: "15938" uid: 192093cf-923b-4607-8d51-3f0edab2a14e kind: Event lastTimestamp: "2026-05-25T13:17:01Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:367463b1d3262f4416182663b2344c09f7ccedc3764563fd3d0aee32ec772aae" already present on machine metadata: creationTimestamp: "2026-05-25T13:17:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:17:01Z" name: switch-graph-339a4-68f7b7cdb-6xr4s.18b2d161fbb14727 namespace: kserve-ci-e2e-test resourceVersion: "15965" uid: 87d7ccda-d5b4-48e3-b53c-e57c3403ce17 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:17:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-339a4} kind: Pod name: switch-graph-339a4-68f7b7cdb-6xr4s namespace: kserve-ci-e2e-test resourceVersion: "15938" uid: 192093cf-923b-4607-8d51-3f0edab2a14e kind: Event lastTimestamp: "2026-05-25T13:17:01Z" message: 'Created container: switch-graph-339a4' metadata: creationTimestamp: "2026-05-25T13:17:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:17:01Z" name: switch-graph-339a4-68f7b7cdb-6xr4s.18b2d161fe7a0886 namespace: kserve-ci-e2e-test resourceVersion: "15966" uid: 063a9434-0038-49bd-b1e8-dbc18a9e489e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:17:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-339a4} kind: Pod name: switch-graph-339a4-68f7b7cdb-6xr4s namespace: kserve-ci-e2e-test resourceVersion: "15938" uid: 192093cf-923b-4607-8d51-3f0edab2a14e kind: Event lastTimestamp: "2026-05-25T13:17:01Z" message: Started container switch-graph-339a4 metadata: creationTimestamp: "2026-05-25T13:17:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:17:01Z" name: switch-graph-339a4-68f7b7cdb-6xr4s.18b2d161feff5946 namespace: kserve-ci-e2e-test resourceVersion: "15967" uid: e167b12a-8fad-4a62-833b-d1ae66fb1878 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:25:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-339a4} kind: Pod name: switch-graph-339a4-68f7b7cdb-6xr4s namespace: kserve-ci-e2e-test resourceVersion: "15938" uid: 192093cf-923b-4607-8d51-3f0edab2a14e kind: Event lastTimestamp: "2026-05-25T13:25:15Z" message: Stopping container switch-graph-339a4 metadata: creationTimestamp: "2026-05-25T13:25:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:25:15Z" name: switch-graph-339a4-68f7b7cdb-6xr4s.18b2d1d508604810 namespace: kserve-ci-e2e-test resourceVersion: "19429" uid: 7e8baaeb-6e47-492e-a53b-fe1029bd350f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T13:25:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-339a4} kind: Pod name: switch-graph-339a4-68f7b7cdb-6xr4s namespace: kserve-ci-e2e-test resourceVersion: "15938" uid: 192093cf-923b-4607-8d51-3f0edab2a14e kind: Event lastTimestamp: "2026-05-25T13:25:43Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-25T13:25:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:25:43Z" name: switch-graph-339a4-68f7b7cdb-6xr4s.18b2d1d594f4b99d namespace: kserve-ci-e2e-test resourceVersion: "19761" uid: a5611b87-cd6c-4dfd-bf90-4bb2c8453ded reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:17:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-339a4-68f7b7cdb namespace: kserve-ci-e2e-test resourceVersion: "15931" uid: ba3eb5a7-b7d8-4102-ad8d-a0f5ed170202 kind: Event lastTimestamp: "2026-05-25T13:17:01Z" message: 'Created pod: switch-graph-339a4-68f7b7cdb-6xr4s' metadata: creationTimestamp: "2026-05-25T13:17:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:17:01Z" name: switch-graph-339a4-68f7b7cdb.18b2d161e05c67b1 namespace: kserve-ci-e2e-test resourceVersion: "15937" uid: ec51af85-448b-4ff2-9b27-00291eddc3ca reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:17:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-339a4 namespace: kserve-ci-e2e-test resourceVersion: "15927" uid: aec21cf5-4837-4efa-947f-91fdb91eda81 kind: Event lastTimestamp: "2026-05-25T13:17:01Z" message: Scaled up replica set switch-graph-339a4-68f7b7cdb from 0 to 1 metadata: creationTimestamp: "2026-05-25T13:17:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:17:01Z" name: switch-graph-339a4.18b2d161df907eca namespace: kserve-ci-e2e-test resourceVersion: "15933" uid: 8f310f67-ff54-46c9-a946-02d43f9d12b3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:17:01Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-339a4 namespace: kserve-ci-e2e-test resourceVersion: "15951" uid: b92a9fb8-8664-4684-815a-3a62f22a677c kind: Event lastTimestamp: "2026-05-25T13:17:01Z" message: InferenceGraph [switch-graph-339a4] is Ready metadata: creationTimestamp: "2026-05-25T13:17:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:17:01Z" name: switch-graph-339a4.18b2d161e4ae8a13 namespace: kserve-ci-e2e-test resourceVersion: "15953" uid: ef9c689f-babc-42ba-b783-959b0ebf751b reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T13:17:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-339a4 namespace: kserve-ci-e2e-test resourceVersion: "15930" uid: a103f52f-bcba-4388-8b4a-65d6e5f8bd04 kind: Event lastTimestamp: "2026-05-25T13:17:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T13:17:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:17:31Z" name: switch-graph-339a4.18b2d1655df696e2 namespace: kserve-ci-e2e-test resourceVersion: "16241" uid: e4de69d8-6c6f-4c78-95d5-18598165dfb6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T13:17:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-339a4 namespace: kserve-ci-e2e-test resourceVersion: "15930" uid: a103f52f-bcba-4388-8b4a-65d6e5f8bd04 kind: Event lastTimestamp: "2026-05-25T13:17:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T13:17:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:17:31Z" name: switch-graph-339a4.18b2d1655e30a45a namespace: kserve-ci-e2e-test resourceVersion: "16245" uid: 06727f9a-1dc0-448c-8f4e-fe300a138f55 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-05-25T13:15:50.676897Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-889c1-858b68998c-h8jts namespace: kserve-ci-e2e-test resourceVersion: "14923" uid: c58bfbf8-1df7-4757-9a67-89b8fb7745df kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-889c1-858b68998c-h8jts to ip-10-0-139-18.ec2.internal metadata: creationTimestamp: "2026-05-25T13:15:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T13:15:50Z" name: switch-graph-889c1-858b68998c-h8jts.18b2d1517c8c7756 namespace: kserve-ci-e2e-test resourceVersion: "14929" uid: 31814f40-a5f9-4f70-acf8-94a6e27d3c16 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7c46b5f76-4gnkb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:15:51Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-889c1-858b68998c-h8jts namespace: kserve-ci-e2e-test resourceVersion: "14932" uid: c58bfbf8-1df7-4757-9a67-89b8fb7745df kind: Event lastTimestamp: "2026-05-25T13:15:51Z" message: Add eth0 [10.132.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T13:15:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T13:15:51Z" name: switch-graph-889c1-858b68998c-h8jts.18b2d1519606cbcb namespace: kserve-ci-e2e-test resourceVersion: "14946" uid: efe4601e-dc62-4ddc-b511-0a52d417c760 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:15:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-889c1} kind: Pod name: switch-graph-889c1-858b68998c-h8jts namespace: kserve-ci-e2e-test resourceVersion: "14925" uid: c58bfbf8-1df7-4757-9a67-89b8fb7745df kind: Event lastTimestamp: "2026-05-25T13:15:51Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:367463b1d3262f4416182663b2344c09f7ccedc3764563fd3d0aee32ec772aae" metadata: creationTimestamp: "2026-05-25T13:15:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:15:51Z" name: switch-graph-889c1-858b68998c-h8jts.18b2d151974d98a6 namespace: kserve-ci-e2e-test resourceVersion: "14948" uid: 80495f5e-1c1d-407b-8004-abedbb4e3987 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:15:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-889c1} kind: Pod name: switch-graph-889c1-858b68998c-h8jts namespace: kserve-ci-e2e-test resourceVersion: "14925" uid: c58bfbf8-1df7-4757-9a67-89b8fb7745df kind: Event lastTimestamp: "2026-05-25T13:15:53Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:367463b1d3262f4416182663b2344c09f7ccedc3764563fd3d0aee32ec772aae" in 2.324s (2.324s including waiting). Image size: 217997404 bytes.' metadata: creationTimestamp: "2026-05-25T13:15:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:15:53Z" name: switch-graph-889c1-858b68998c-h8jts.18b2d15221d73854 namespace: kserve-ci-e2e-test resourceVersion: "14966" uid: 6109ceea-314e-4708-b833-caffda2af085 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:15:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-889c1} kind: Pod name: switch-graph-889c1-858b68998c-h8jts namespace: kserve-ci-e2e-test resourceVersion: "14925" uid: c58bfbf8-1df7-4757-9a67-89b8fb7745df kind: Event lastTimestamp: "2026-05-25T13:15:53Z" message: 'Created container: switch-graph-889c1' metadata: creationTimestamp: "2026-05-25T13:15:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:15:53Z" name: switch-graph-889c1-858b68998c-h8jts.18b2d152253b0da2 namespace: kserve-ci-e2e-test resourceVersion: "14968" uid: fe6fcf16-963b-4c8e-91a9-f6d4131e89cb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:15:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-889c1} kind: Pod name: switch-graph-889c1-858b68998c-h8jts namespace: kserve-ci-e2e-test resourceVersion: "14925" uid: c58bfbf8-1df7-4757-9a67-89b8fb7745df kind: Event lastTimestamp: "2026-05-25T13:15:53Z" message: Started container switch-graph-889c1 metadata: creationTimestamp: "2026-05-25T13:15:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:15:53Z" name: switch-graph-889c1-858b68998c-h8jts.18b2d15225b11c47 namespace: kserve-ci-e2e-test resourceVersion: "14969" uid: ee787e27-99bd-48cd-ad89-2b2424a7fa11 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:16:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-889c1} kind: Pod name: switch-graph-889c1-858b68998c-h8jts namespace: kserve-ci-e2e-test resourceVersion: "14925" uid: c58bfbf8-1df7-4757-9a67-89b8fb7745df kind: Event lastTimestamp: "2026-05-25T13:16:00Z" message: Stopping container switch-graph-889c1 metadata: creationTimestamp: "2026-05-25T13:16:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:16:00Z" name: switch-graph-889c1-858b68998c-h8jts.18b2d153d84a5ecc namespace: kserve-ci-e2e-test resourceVersion: "15061" uid: 5161d8f5-fe70-4b9a-9201-a2f9aea5bf08 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T13:16:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-889c1} kind: Pod name: switch-graph-889c1-858b68998c-h8jts namespace: kserve-ci-e2e-test resourceVersion: "14925" uid: c58bfbf8-1df7-4757-9a67-89b8fb7745df kind: Event lastTimestamp: "2026-05-25T13:16:29Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-25T13:16:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:16:29Z" name: switch-graph-889c1-858b68998c-h8jts.18b2d154c5992dd0 namespace: kserve-ci-e2e-test resourceVersion: "15477" uid: 0d0ec448-a3e7-483a-8893-186584ea838a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-18.ec2.internal source: component: kubelet host: ip-10-0-139-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:15:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-889c1-858b68998c namespace: kserve-ci-e2e-test resourceVersion: "14919" uid: da8fbfbe-843c-450f-9674-339db5ce185e kind: Event lastTimestamp: "2026-05-25T13:15:50Z" message: 'Created pod: switch-graph-889c1-858b68998c-h8jts' metadata: creationTimestamp: "2026-05-25T13:15:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:15:50Z" name: switch-graph-889c1-858b68998c.18b2d1517c1a7216 namespace: kserve-ci-e2e-test resourceVersion: "14924" uid: 671b4b9f-9f60-47b4-b6bd-50bb56a913d4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:15:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-889c1 namespace: kserve-ci-e2e-test resourceVersion: "14917" uid: 2309424c-c990-4c30-9827-ca32505faf96 kind: Event lastTimestamp: "2026-05-25T13:15:50Z" message: Scaled up replica set switch-graph-889c1-858b68998c from 0 to 1 metadata: creationTimestamp: "2026-05-25T13:15:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T13:15:50Z" name: switch-graph-889c1.18b2d1517b437f52 namespace: kserve-ci-e2e-test resourceVersion: "14920" uid: d11a4f22-f913-4c8d-ad0b-8669251055f7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:15:50Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-889c1 namespace: kserve-ci-e2e-test resourceVersion: "14934" uid: f1be53e6-4b02-459c-94ff-57b1e9dc59e2 kind: Event lastTimestamp: "2026-05-25T13:15:50Z" message: InferenceGraph [switch-graph-889c1] is Ready metadata: creationTimestamp: "2026-05-25T13:15:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:15:50Z" name: switch-graph-889c1.18b2d15180bc2e65 namespace: kserve-ci-e2e-test resourceVersion: "14936" uid: 7e716504-7f89-4b98-b761-120a47d9a13c reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:15:50Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-889c1 namespace: kserve-ci-e2e-test resourceVersion: "14910" uid: f1be53e6-4b02-459c-94ff-57b1e9dc59e2 kind: Event lastTimestamp: "2026-05-25T13:15:50Z" message: 'Failed to update status for InferenceGraph "switch-graph-889c1": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-889c1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T13:15:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:15:50Z" name: switch-graph-889c1.18b2d15183243c49 namespace: kserve-ci-e2e-test resourceVersion: "14937" uid: 31e93fd8-8530-4f3a-adc0-b0b475dcd5e3 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:15:50Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-889c1 namespace: kserve-ci-e2e-test resourceVersion: "14910" uid: f1be53e6-4b02-459c-94ff-57b1e9dc59e2 kind: Event lastTimestamp: "2026-05-25T13:15:50Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-889c1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T13:15:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T13:15:50Z" name: switch-graph-889c1.18b2d15183249f87 namespace: kserve-ci-e2e-test resourceVersion: "14938" uid: d709225a-93d5-4580-93fa-3614aa24084c reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning kind: EventList metadata: resourceVersion: "33360"