--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:07:31.193250Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-a3448-6fb64b7f46-nrcwp namespace: kserve-ci-e2e-test resourceVersion: "21656" uid: 65f08c98-48f8-485d-ac4f-004d99fb4972 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-a3448-6fb64b7f46-nrcwp to ip-10-0-129-152.ec2.internal metadata: creationTimestamp: "2026-09-16T19:07:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:07:31Z" name: ensemble-graph-a3448-6fb64b7f46-nrcwp.18d5e2ab0a82af80 namespace: kserve-ci-e2e-test resourceVersion: "21662" uid: 0d4f4962-92e8-4fe9-8a6f-582072a015c9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-594cc48779-9d5dg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:31Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-a3448-6fb64b7f46-nrcwp namespace: kserve-ci-e2e-test resourceVersion: "21665" uid: 65f08c98-48f8-485d-ac4f-004d99fb4972 kind: Event lastTimestamp: "2026-09-16T19:07:31Z" message: Add eth0 [10.133.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:07:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:07:31Z" name: ensemble-graph-a3448-6fb64b7f46-nrcwp.18d5e2ab2cc4fd3f namespace: kserve-ci-e2e-test resourceVersion: "21677" uid: bc7a7ce1-8bd3-4f90-ae19-c30453c89525 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-a3448} kind: Pod name: ensemble-graph-a3448-6fb64b7f46-nrcwp namespace: kserve-ci-e2e-test resourceVersion: "21657" uid: 65f08c98-48f8-485d-ac4f-004d99fb4972 kind: Event lastTimestamp: "2026-09-16T19:07:31Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:10ee05d4acb8a93a3eeee1c000e44a3fdd4ba5c5852a56e3b2d09fe8b08cf42b" already present on machine metadata: creationTimestamp: "2026-09-16T19:07:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:31Z" name: ensemble-graph-a3448-6fb64b7f46-nrcwp.18d5e2ab2e0f4abc namespace: kserve-ci-e2e-test resourceVersion: "21679" uid: 4fe741ed-ff3b-4910-a5ea-e794c11defe9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-152.ec2.internal source: component: kubelet host: ip-10-0-129-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-a3448} kind: Pod name: ensemble-graph-a3448-6fb64b7f46-nrcwp namespace: kserve-ci-e2e-test resourceVersion: "21657" uid: 65f08c98-48f8-485d-ac4f-004d99fb4972 kind: Event lastTimestamp: "2026-09-16T19:07:31Z" message: 'Created container: ensemble-graph-a3448' metadata: creationTimestamp: "2026-09-16T19:07:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:31Z" name: ensemble-graph-a3448-6fb64b7f46-nrcwp.18d5e2ab31422afd namespace: kserve-ci-e2e-test resourceVersion: "21681" uid: 7c61f4a7-d016-44f1-bf81-fe50ec2827f7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-152.ec2.internal source: component: kubelet host: ip-10-0-129-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-a3448} kind: Pod name: ensemble-graph-a3448-6fb64b7f46-nrcwp namespace: kserve-ci-e2e-test resourceVersion: "21657" uid: 65f08c98-48f8-485d-ac4f-004d99fb4972 kind: Event lastTimestamp: "2026-09-16T19:07:31Z" message: Started container ensemble-graph-a3448 metadata: creationTimestamp: "2026-09-16T19:07:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:31Z" name: ensemble-graph-a3448-6fb64b7f46-nrcwp.18d5e2ab31b08bcc namespace: kserve-ci-e2e-test resourceVersion: "21683" uid: 5e537a4a-3ff4-4148-a0c2-a0f564beee6b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-152.ec2.internal source: component: kubelet host: ip-10-0-129-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:15:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-a3448} kind: Pod name: ensemble-graph-a3448-6fb64b7f46-nrcwp namespace: kserve-ci-e2e-test resourceVersion: "21657" uid: 65f08c98-48f8-485d-ac4f-004d99fb4972 kind: Event lastTimestamp: "2026-09-16T19:15:45Z" message: Stopping container ensemble-graph-a3448 metadata: creationTimestamp: "2026-09-16T19:15:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:15:45Z" name: ensemble-graph-a3448-6fb64b7f46-nrcwp.18d5e31e33c04e3a namespace: kserve-ci-e2e-test resourceVersion: "25178" uid: cfa865ec-7e1e-4870-b495-caa091226593 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-152.ec2.internal source: component: kubelet host: ip-10-0-129-152.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T19:15:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-a3448} kind: Pod name: ensemble-graph-a3448-6fb64b7f46-nrcwp namespace: kserve-ci-e2e-test resourceVersion: "21657" uid: 65f08c98-48f8-485d-ac4f-004d99fb4972 kind: Event lastTimestamp: "2026-09-16T19:16:13Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-16T19:15:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:16:13Z" name: ensemble-graph-a3448-6fb64b7f46-nrcwp.18d5e31eda3a17d9 namespace: kserve-ci-e2e-test resourceVersion: "25519" uid: 4d768eb0-22de-4c0b-b882-ec94dc7427d7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-152.ec2.internal source: component: kubelet host: ip-10-0-129-152.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-a3448-6fb64b7f46 namespace: kserve-ci-e2e-test resourceVersion: "21652" uid: 46de51fc-dc81-49f1-9e50-1f4fb6ea89a2 kind: Event lastTimestamp: "2026-09-16T19:07:31Z" message: 'Created pod: ensemble-graph-a3448-6fb64b7f46-nrcwp' metadata: creationTimestamp: "2026-09-16T19:07:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:07:31Z" name: ensemble-graph-a3448-6fb64b7f46.18d5e2ab0a092010 namespace: kserve-ci-e2e-test resourceVersion: "21658" uid: 61283743-ec6d-4b8d-8f3a-c48f863c45d2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-a3448 namespace: kserve-ci-e2e-test resourceVersion: "21650" uid: 2bd98159-97d0-4c5f-ac64-c1f54660224c kind: Event lastTimestamp: "2026-09-16T19:07:31Z" message: Scaled up replica set ensemble-graph-a3448-6fb64b7f46 from 0 to 1 metadata: creationTimestamp: "2026-09-16T19:07:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:07:31Z" name: ensemble-graph-a3448.18d5e2ab09125b1a namespace: kserve-ci-e2e-test resourceVersion: "21654" uid: 6319e5b4-6bb0-4274-9ca2-ac3a5b86ab1e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:31Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-a3448 namespace: kserve-ci-e2e-test resourceVersion: "21669" uid: e090a9d4-8b21-4254-8cee-99dfaf9d9122 kind: Event lastTimestamp: "2026-09-16T19:07:31Z" message: InferenceGraph [ensemble-graph-a3448] is Ready metadata: creationTimestamp: "2026-09-16T19:07:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:07:31Z" name: ensemble-graph-a3448.18d5e2ab0fa51ebc namespace: kserve-ci-e2e-test resourceVersion: "21671" uid: 037e86b0-95d0-4b5c-970d-a7eb2f119737 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-a3448 namespace: kserve-ci-e2e-test resourceVersion: "21651" uid: bb8fd3c3-e70d-4fce-a7f9-bbffcf2ffae4 kind: Event lastTimestamp: "2026-09-16T19:07:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:07:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:07:46Z" name: ensemble-graph-a3448.18d5e2ae8783a6eb namespace: kserve-ci-e2e-test resourceVersion: "21806" uid: b5262b1e-6209-40c6-a34b-91b7aa1e89b0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-a3448 namespace: kserve-ci-e2e-test resourceVersion: "21651" uid: bb8fd3c3-e70d-4fce-a7f9-bbffcf2ffae4 kind: Event lastTimestamp: "2026-09-16T19:07:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:07:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:07:46Z" name: ensemble-graph-a3448.18d5e2ae87e19f87 namespace: kserve-ci-e2e-test resourceVersion: "21807" uid: 77e19ede-2771-4597-8ddb-fbfa0dfc10e8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-a3448 namespace: kserve-ci-e2e-test resourceVersion: "21805" uid: bb8fd3c3-e70d-4fce-a7f9-bbffcf2ffae4 kind: Event lastTimestamp: "2026-09-16T19:08:01Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:08:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:08:01Z" name: ensemble-graph-a3448.18d5e2b206b27441 namespace: kserve-ci-e2e-test resourceVersion: "21933" uid: 3f42bb1f-edc8-403d-a9ad-e6ee5e7e01d9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-a3448 namespace: kserve-ci-e2e-test resourceVersion: "21805" uid: bb8fd3c3-e70d-4fce-a7f9-bbffcf2ffae4 kind: Event lastTimestamp: "2026-09-16T19:08:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:08:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:08:01Z" name: ensemble-graph-a3448.18d5e2b207628123 namespace: kserve-ci-e2e-test resourceVersion: "21935" uid: e8906a89-e525-416f-b472-9765c902a69c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:06:20.766132Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-c6de0-db6f655-4zjm6 namespace: kserve-ci-e2e-test resourceVersion: "20672" uid: 6a0f5bba-2715-4664-b25b-d9574da30905 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-c6de0-db6f655-4zjm6 to ip-10-0-129-152.ec2.internal metadata: creationTimestamp: "2026-09-16T19:06:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:06:20Z" name: ensemble-graph-c6de0-db6f655-4zjm6.18d5e29aa4ba0cc5 namespace: kserve-ci-e2e-test resourceVersion: "20678" uid: bae176c6-948f-4767-8259-5b65a702e914 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-594cc48779-9d5dg source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:06:20Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-c6de0-db6f655-4zjm6 namespace: kserve-ci-e2e-test resourceVersion: "20674" uid: 6a0f5bba-2715-4664-b25b-d9574da30905 kind: Event lastTimestamp: "2026-09-16T19:06:21Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-c6de0-serving-cert" not found' metadata: creationTimestamp: "2026-09-16T19:06:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:06:21Z" name: ensemble-graph-c6de0-db6f655-4zjm6.18d5e29aaff59cce namespace: kserve-ci-e2e-test resourceVersion: "20698" uid: 1c786e93-de0a-4f57-afe4-3fee9a6f9fbc reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-152.ec2.internal source: component: kubelet host: ip-10-0-129-152.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:22Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-c6de0-db6f655-4zjm6 namespace: kserve-ci-e2e-test resourceVersion: "20684" uid: 6a0f5bba-2715-4664-b25b-d9574da30905 kind: Event lastTimestamp: "2026-09-16T19:06:22Z" message: Add eth0 [10.133.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:06:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:06:22Z" name: ensemble-graph-c6de0-db6f655-4zjm6.18d5e29b1ffba23c namespace: kserve-ci-e2e-test resourceVersion: "20713" uid: 937f9aa9-5a40-4682-a040-4051e6e0049d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c6de0} kind: Pod name: ensemble-graph-c6de0-db6f655-4zjm6 namespace: kserve-ci-e2e-test resourceVersion: "20674" uid: 6a0f5bba-2715-4664-b25b-d9574da30905 kind: Event lastTimestamp: "2026-09-16T19:06:22Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:10ee05d4acb8a93a3eeee1c000e44a3fdd4ba5c5852a56e3b2d09fe8b08cf42b" already present on machine metadata: creationTimestamp: "2026-09-16T19:06:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:06:22Z" name: ensemble-graph-c6de0-db6f655-4zjm6.18d5e29b213e1b6a namespace: kserve-ci-e2e-test resourceVersion: "20715" uid: 9a9caa8f-bdde-4284-888b-f338a01b0976 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-152.ec2.internal source: component: kubelet host: ip-10-0-129-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c6de0} kind: Pod name: ensemble-graph-c6de0-db6f655-4zjm6 namespace: kserve-ci-e2e-test resourceVersion: "20674" uid: 6a0f5bba-2715-4664-b25b-d9574da30905 kind: Event lastTimestamp: "2026-09-16T19:06:22Z" message: 'Created container: ensemble-graph-c6de0' metadata: creationTimestamp: "2026-09-16T19:06:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:06:22Z" name: ensemble-graph-c6de0-db6f655-4zjm6.18d5e29b243e90e1 namespace: kserve-ci-e2e-test resourceVersion: "20717" uid: 5577af44-4843-438d-8b12-eb76daeb16be reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-152.ec2.internal source: component: kubelet host: ip-10-0-129-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c6de0} kind: Pod name: ensemble-graph-c6de0-db6f655-4zjm6 namespace: kserve-ci-e2e-test resourceVersion: "20674" uid: 6a0f5bba-2715-4664-b25b-d9574da30905 kind: Event lastTimestamp: "2026-09-16T19:06:22Z" message: Started container ensemble-graph-c6de0 metadata: creationTimestamp: "2026-09-16T19:06:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:06:22Z" name: ensemble-graph-c6de0-db6f655-4zjm6.18d5e29b24c06313 namespace: kserve-ci-e2e-test resourceVersion: "20719" uid: dcd0fd30-40a8-4b3e-9ef2-6dd29bfa1f89 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-152.ec2.internal source: component: kubelet host: ip-10-0-129-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c6de0} kind: Pod name: ensemble-graph-c6de0-db6f655-4zjm6 namespace: kserve-ci-e2e-test resourceVersion: "20674" uid: 6a0f5bba-2715-4664-b25b-d9574da30905 kind: Event lastTimestamp: "2026-09-16T19:06:30Z" message: Stopping container ensemble-graph-c6de0 metadata: creationTimestamp: "2026-09-16T19:06:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:06:30Z" name: ensemble-graph-c6de0-db6f655-4zjm6.18d5e29cfd55b79b namespace: kserve-ci-e2e-test resourceVersion: "20800" uid: dfd25cc6-8bc8-44af-bfb4-548dc4eed2a0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-152.ec2.internal source: component: kubelet host: ip-10-0-129-152.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T19:06:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c6de0} kind: Pod name: ensemble-graph-c6de0-db6f655-4zjm6 namespace: kserve-ci-e2e-test resourceVersion: "20674" uid: 6a0f5bba-2715-4664-b25b-d9574da30905 kind: Event lastTimestamp: "2026-09-16T19:06:59Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-16T19:06:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:06:59Z" name: ensemble-graph-c6de0-db6f655-4zjm6.18d5e29dc9e20998 namespace: kserve-ci-e2e-test resourceVersion: "21182" uid: 834dd255-7fff-4b57-a897-41c3d4ddd052 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-152.ec2.internal source: component: kubelet host: ip-10-0-129-152.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-c6de0-db6f655 namespace: kserve-ci-e2e-test resourceVersion: "20669" uid: 37238b67-6bfe-4eca-bb64-42e2969167c3 kind: Event lastTimestamp: "2026-09-16T19:06:20Z" message: 'Created pod: ensemble-graph-c6de0-db6f655-4zjm6' metadata: creationTimestamp: "2026-09-16T19:06:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:06:20Z" name: ensemble-graph-c6de0-db6f655.18d5e29aa41a82ab namespace: kserve-ci-e2e-test resourceVersion: "20675" uid: 254e4fed-825e-41c3-838b-3d80ffc92bfb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-c6de0 namespace: kserve-ci-e2e-test resourceVersion: "20667" uid: e5bd4eac-c9bf-444f-bdb4-5e7118d08f98 kind: Event lastTimestamp: "2026-09-16T19:06:20Z" message: Scaled up replica set ensemble-graph-c6de0-db6f655 from 0 to 1 metadata: creationTimestamp: "2026-09-16T19:06:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:06:20Z" name: ensemble-graph-c6de0.18d5e29aa34daee3 namespace: kserve-ci-e2e-test resourceVersion: "20671" uid: e207071f-e8d8-4363-88b2-b0c8600dd7cd reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:20Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-c6de0 namespace: kserve-ci-e2e-test resourceVersion: "20686" uid: 38bd9dcd-e5bc-4c70-b0e1-3b3f4162bb72 kind: Event lastTimestamp: "2026-09-16T19:06:20Z" message: InferenceGraph [ensemble-graph-c6de0] is Ready metadata: creationTimestamp: "2026-09-16T19:06:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:06:20Z" name: ensemble-graph-c6de0.18d5e29aa98c78bc namespace: kserve-ci-e2e-test resourceVersion: "20688" uid: aa0d467b-02fb-4bf3-8024-48900ffa4185 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T18:56:46.037989Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-2796f-predictor-cc7c89b74-d9k44 namespace: kserve-ci-e2e-test resourceVersion: "16158" uid: 9452c4a7-5a7a-444a-800f-d1d8695c0140 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-2796f-predictor-cc7c89b74-d9k44 to ip-10-0-138-79.ec2.internal metadata: creationTimestamp: "2026-09-16T18:56:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T18:56:46Z" name: error-404-isvc-2796f-predictor-cc7c89b74-d9k44.18d5e214d4429023 namespace: kserve-ci-e2e-test resourceVersion: "16162" uid: 9701842b-7639-4565-aba6-6ae89faf924a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-594cc48779-9d5dg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:46Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-2796f-predictor-cc7c89b74-d9k44 namespace: kserve-ci-e2e-test resourceVersion: "16167" uid: 9452c4a7-5a7a-444a-800f-d1d8695c0140 kind: Event lastTimestamp: "2026-09-16T18:56:46Z" message: Add eth0 [10.132.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T18:56:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T18:56:46Z" name: error-404-isvc-2796f-predictor-cc7c89b74-d9k44.18d5e214f6786c3b namespace: kserve-ci-e2e-test resourceVersion: "16184" uid: b4069899-3601-4eec-bf08-fe1fa9d20e3c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2796f-predictor-cc7c89b74-d9k44 namespace: kserve-ci-e2e-test resourceVersion: "16160" uid: 9452c4a7-5a7a-444a-800f-d1d8695c0140 kind: Event lastTimestamp: "2026-09-16T18:56:46Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1986" already present on machine metadata: creationTimestamp: "2026-09-16T18:56:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:46Z" name: error-404-isvc-2796f-predictor-cc7c89b74-d9k44.18d5e214f7f9d596 namespace: kserve-ci-e2e-test resourceVersion: "16186" uid: 74b7c823-e4f2-436a-b2c3-41579b7f76ab reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2796f-predictor-cc7c89b74-d9k44 namespace: kserve-ci-e2e-test resourceVersion: "16160" uid: 9452c4a7-5a7a-444a-800f-d1d8695c0140 kind: Event lastTimestamp: "2026-09-16T18:56:46Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T18:56:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:46Z" name: error-404-isvc-2796f-predictor-cc7c89b74-d9k44.18d5e214fa88bbc7 namespace: kserve-ci-e2e-test resourceVersion: "16187" uid: d686cf3d-63a3-421c-b85d-30864d970e89 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2796f-predictor-cc7c89b74-d9k44 namespace: kserve-ci-e2e-test resourceVersion: "16160" uid: 9452c4a7-5a7a-444a-800f-d1d8695c0140 kind: Event lastTimestamp: "2026-09-16T18:56:46Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T18:56:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:46Z" name: error-404-isvc-2796f-predictor-cc7c89b74-d9k44.18d5e214fafc848b namespace: kserve-ci-e2e-test resourceVersion: "16188" uid: 60f4ff4f-eced-4850-9689-8cd3cdbceee1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2796f-predictor-cc7c89b74-d9k44 namespace: kserve-ci-e2e-test resourceVersion: "16160" uid: 9452c4a7-5a7a-444a-800f-d1d8695c0140 kind: Event lastTimestamp: "2026-09-16T18:56:46Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-16T18:56:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:46Z" name: error-404-isvc-2796f-predictor-cc7c89b74-d9k44.18d5e214fb05cf00 namespace: kserve-ci-e2e-test resourceVersion: "16189" uid: c4034768-78bd-433c-8be5-53249ea009b5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2796f-predictor-cc7c89b74-d9k44 namespace: kserve-ci-e2e-test resourceVersion: "16160" uid: 9452c4a7-5a7a-444a-800f-d1d8695c0140 kind: Event lastTimestamp: "2026-09-16T18:56:46Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T18:56:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:46Z" name: error-404-isvc-2796f-predictor-cc7c89b74-d9k44.18d5e214fd6d7e1d namespace: kserve-ci-e2e-test resourceVersion: "16190" uid: e37f7d3b-6262-41c0-a91c-db8613ab8e53 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2796f-predictor-cc7c89b74-d9k44 namespace: kserve-ci-e2e-test resourceVersion: "16160" uid: 9452c4a7-5a7a-444a-800f-d1d8695c0140 kind: Event lastTimestamp: "2026-09-16T18:56:46Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T18:56:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:46Z" name: error-404-isvc-2796f-predictor-cc7c89b74-d9k44.18d5e214fddf5308 namespace: kserve-ci-e2e-test resourceVersion: "16192" uid: 9ea3873d-aa57-494f-88d9-86ff35d624b1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T18:56:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2796f-predictor-cc7c89b74-d9k44 namespace: kserve-ci-e2e-test resourceVersion: "16160" uid: 9452c4a7-5a7a-444a-800f-d1d8695c0140 kind: Event lastTimestamp: "2026-09-16T18:57:24Z" message: 'Readiness probe failed: dial tcp 10.132.0.20:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T18:56:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:57:24Z" name: error-404-isvc-2796f-predictor-cc7c89b74-d9k44.18d5e21565063f4e namespace: kserve-ci-e2e-test resourceVersion: "16584" uid: 7b1c8296-7f56-43e6-a9b1-5b975c5f443c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2796f-predictor-cc7c89b74-d9k44 namespace: kserve-ci-e2e-test resourceVersion: "16160" uid: 9452c4a7-5a7a-444a-800f-d1d8695c0140 kind: Event lastTimestamp: "2026-09-16T19:06:00Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T19:06:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:06:00Z" name: error-404-isvc-2796f-predictor-cc7c89b74-d9k44.18d5e295f7e1bab5 namespace: kserve-ci-e2e-test resourceVersion: "20396" uid: 5ef9e5ce-e3d0-48f3-99cb-2516626af166 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2796f-predictor-cc7c89b74-d9k44 namespace: kserve-ci-e2e-test resourceVersion: "16160" uid: 9452c4a7-5a7a-444a-800f-d1d8695c0140 kind: Event lastTimestamp: "2026-09-16T19:06:00Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:06:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:06:00Z" name: error-404-isvc-2796f-predictor-cc7c89b74-d9k44.18d5e295f7e26bad namespace: kserve-ci-e2e-test resourceVersion: "20398" uid: 40d539dd-83e1-4720-8ef8-f6a78323f965 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:46Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-2796f-predictor-cc7c89b74 namespace: kserve-ci-e2e-test resourceVersion: "16152" uid: 5f79e219-e6b7-43fa-8895-a3f723915e68 kind: Event lastTimestamp: "2026-09-16T18:56:46Z" message: 'Created pod: error-404-isvc-2796f-predictor-cc7c89b74-d9k44' metadata: creationTimestamp: "2026-09-16T18:56:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:56:46Z" name: error-404-isvc-2796f-predictor-cc7c89b74.18d5e214d3af73f0 namespace: kserve-ci-e2e-test resourceVersion: "16159" uid: 909e5716-b432-4854-bc76-e93542de051d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:46Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-2796f-predictor namespace: kserve-ci-e2e-test resourceVersion: "16150" uid: e29a1892-8fe7-45c8-a44b-b2e25d832a86 kind: Event lastTimestamp: "2026-09-16T18:56:46Z" message: Scaled up replica set error-404-isvc-2796f-predictor-cc7c89b74 from 0 to 1 metadata: creationTimestamp: "2026-09-16T18:56:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:56:46Z" name: error-404-isvc-2796f-predictor.18d5e214d231385a namespace: kserve-ci-e2e-test resourceVersion: "16153" uid: 643d9149-0bd2-4d22-848a-260b0a07eb9d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T18:57:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2796f-predictor namespace: kserve-ci-e2e-test resourceVersion: "16151" uid: c3be1257-9191-4233-9aff-92a93df1ba15 kind: Event lastTimestamp: "2026-09-16T18:57:16Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T18:57:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:57:16Z" name: error-404-isvc-2796f-predictor.18d5e21850bcca81 namespace: kserve-ci-e2e-test resourceVersion: "16515" uid: ea1b4c2f-6d16-4398-9659-5cce0f1a2f08 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T18:57:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2796f-predictor namespace: kserve-ci-e2e-test resourceVersion: "16151" uid: c3be1257-9191-4233-9aff-92a93df1ba15 kind: Event lastTimestamp: "2026-09-16T18:57:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T18:57:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:57:16Z" name: error-404-isvc-2796f-predictor.18d5e218512738ae namespace: kserve-ci-e2e-test resourceVersion: "16516" uid: 7ca6a3ba-ed9c-4668-a7d3-52d224df701b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T18:57:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2796f-predictor namespace: kserve-ci-e2e-test resourceVersion: "16359" uid: c3be1257-9191-4233-9aff-92a93df1ba15 kind: Event lastTimestamp: "2026-09-16T18:58:01Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T18:57:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:58:01Z" name: error-404-isvc-2796f-predictor.18d5e21f4f31e7a5 namespace: kserve-ci-e2e-test resourceVersion: "16912" uid: 6e34e8ff-4972-452c-b1a1-605f50c8eeab reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T18:57:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2796f-predictor namespace: kserve-ci-e2e-test resourceVersion: "16359" uid: c3be1257-9191-4233-9aff-92a93df1ba15 kind: Event lastTimestamp: "2026-09-16T18:58:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T18:57:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:58:01Z" name: error-404-isvc-2796f-predictor.18d5e21f4ff33c82 namespace: kserve-ci-e2e-test resourceVersion: "16913" uid: 752012db-f57e-48d6-9f99-4a4ea8d21460 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-16T18:56:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2796f namespace: kserve-ci-e2e-test resourceVersion: "16140" uid: 41dcd8d7-5dbf-488c-90a7-50065a861d6e kind: Event lastTimestamp: "2026-09-16T18:57:34Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T18:56:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T18:57:34Z" name: error-404-isvc-2796f.18d5e214d2362f50 namespace: kserve-ci-e2e-test resourceVersion: "16675" uid: 4c217b8e-4a34-4a07-8d91-e07b97f1b61f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2796f namespace: kserve-ci-e2e-test resourceVersion: "16673" uid: 41dcd8d7-5dbf-488c-90a7-50065a861d6e kind: Event lastTimestamp: "2026-09-16T18:57:34Z" message: InferenceService [error-404-isvc-2796f] is Ready metadata: creationTimestamp: "2026-09-16T18:57:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T18:57:34Z" name: error-404-isvc-2796f.18d5e2202010d9e9 namespace: kserve-ci-e2e-test resourceVersion: "16674" uid: 6c07ad4e-3e5a-4d7a-9021-cc0516091505 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:16:56.519288Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4a90a-predictor-6759885b-smchc namespace: kserve-ci-e2e-test resourceVersion: "26455" uid: 33ae2c1d-7a06-43ed-ae65-5b3f2a38ba11 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-4a90a-predictor-6759885b-smchc to ip-10-0-138-79.ec2.internal metadata: creationTimestamp: "2026-09-16T19:16:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:16:56Z" name: error-404-isvc-4a90a-predictor-6759885b-smchc.18d5e32eaa91581b namespace: kserve-ci-e2e-test resourceVersion: "26460" uid: d2c7c126-1c92-405d-a342-44ed56b9ea70 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-594cc48779-9d5dg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:57Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4a90a-predictor-6759885b-smchc namespace: kserve-ci-e2e-test resourceVersion: "26466" uid: 33ae2c1d-7a06-43ed-ae65-5b3f2a38ba11 kind: Event lastTimestamp: "2026-09-16T19:16:57Z" message: Add eth0 [10.132.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:16:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:16:57Z" name: error-404-isvc-4a90a-predictor-6759885b-smchc.18d5e32ecc5a7312 namespace: kserve-ci-e2e-test resourceVersion: "26482" uid: f2880a28-31fe-40a6-974c-b4bf54bba5b3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4a90a-predictor-6759885b-smchc namespace: kserve-ci-e2e-test resourceVersion: "26458" uid: 33ae2c1d-7a06-43ed-ae65-5b3f2a38ba11 kind: Event lastTimestamp: "2026-09-16T19:16:57Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1986" already present on machine metadata: creationTimestamp: "2026-09-16T19:16:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:16:57Z" name: error-404-isvc-4a90a-predictor-6759885b-smchc.18d5e32ecddb09c1 namespace: kserve-ci-e2e-test resourceVersion: "26485" uid: 0735a99c-7a00-4efd-806e-83b00123a14b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4a90a-predictor-6759885b-smchc namespace: kserve-ci-e2e-test resourceVersion: "26458" uid: 33ae2c1d-7a06-43ed-ae65-5b3f2a38ba11 kind: Event lastTimestamp: "2026-09-16T19:16:57Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T19:16:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:16:57Z" name: error-404-isvc-4a90a-predictor-6759885b-smchc.18d5e32ed17284f6 namespace: kserve-ci-e2e-test resourceVersion: "26487" uid: d37a68e3-4310-420c-a000-908bf1214b2c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4a90a-predictor-6759885b-smchc namespace: kserve-ci-e2e-test resourceVersion: "26458" uid: 33ae2c1d-7a06-43ed-ae65-5b3f2a38ba11 kind: Event lastTimestamp: "2026-09-16T19:16:57Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T19:16:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:16:57Z" name: error-404-isvc-4a90a-predictor-6759885b-smchc.18d5e32ed1ee3264 namespace: kserve-ci-e2e-test resourceVersion: "26488" uid: f11daa5c-0ed3-4ecf-b94a-bc80090c59c5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4a90a-predictor-6759885b-smchc namespace: kserve-ci-e2e-test resourceVersion: "26458" uid: 33ae2c1d-7a06-43ed-ae65-5b3f2a38ba11 kind: Event lastTimestamp: "2026-09-16T19:16:57Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-16T19:16:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:16:57Z" name: error-404-isvc-4a90a-predictor-6759885b-smchc.18d5e32ed1f7ba39 namespace: kserve-ci-e2e-test resourceVersion: "26489" uid: c461da43-d301-406a-b7ef-ac027a7a0b3b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4a90a-predictor-6759885b-smchc namespace: kserve-ci-e2e-test resourceVersion: "26458" uid: 33ae2c1d-7a06-43ed-ae65-5b3f2a38ba11 kind: Event lastTimestamp: "2026-09-16T19:16:57Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T19:16:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:16:57Z" name: error-404-isvc-4a90a-predictor-6759885b-smchc.18d5e32ed4b6353b namespace: kserve-ci-e2e-test resourceVersion: "26491" uid: fc5d295e-f343-4fcc-9e7e-2923da4d905d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4a90a-predictor-6759885b-smchc namespace: kserve-ci-e2e-test resourceVersion: "26458" uid: 33ae2c1d-7a06-43ed-ae65-5b3f2a38ba11 kind: Event lastTimestamp: "2026-09-16T19:16:57Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:16:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:16:57Z" name: error-404-isvc-4a90a-predictor-6759885b-smchc.18d5e32ed52d5bb2 namespace: kserve-ci-e2e-test resourceVersion: "26492" uid: b5845ac7-5af9-4176-b99a-de5233539692 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-16T19:16:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4a90a-predictor-6759885b-smchc namespace: kserve-ci-e2e-test resourceVersion: "26458" uid: 33ae2c1d-7a06-43ed-ae65-5b3f2a38ba11 kind: Event lastTimestamp: "2026-09-16T19:26:13Z" message: 'Readiness probe failed: dial tcp 10.132.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:16:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:26:13Z" name: error-404-isvc-4a90a-predictor-6759885b-smchc.18d5e32ef39a62f8 namespace: kserve-ci-e2e-test resourceVersion: "30573" uid: f0278c59-003d-4a4d-ac19-1900245419cc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:26:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4a90a-predictor-6759885b-smchc namespace: kserve-ci-e2e-test resourceVersion: "26458" uid: 33ae2c1d-7a06-43ed-ae65-5b3f2a38ba11 kind: Event lastTimestamp: "2026-09-16T19:26:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T19:26:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:26:11Z" name: error-404-isvc-4a90a-predictor-6759885b-smchc.18d5e3afcc12b9c2 namespace: kserve-ci-e2e-test resourceVersion: "30551" uid: 09d52888-fd13-455f-8c8e-f02ad44e64fb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:26:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4a90a-predictor-6759885b-smchc namespace: kserve-ci-e2e-test resourceVersion: "26458" uid: 33ae2c1d-7a06-43ed-ae65-5b3f2a38ba11 kind: Event lastTimestamp: "2026-09-16T19:26:11Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:26:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:26:11Z" name: error-404-isvc-4a90a-predictor-6759885b-smchc.18d5e3afcc134760 namespace: kserve-ci-e2e-test resourceVersion: "30552" uid: 836a5b46-9287-44ea-a1c4-42ce36a7f608 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:26:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4a90a-predictor-6759885b-smchc namespace: kserve-ci-e2e-test resourceVersion: "26458" uid: 33ae2c1d-7a06-43ed-ae65-5b3f2a38ba11 kind: Event lastTimestamp: "2026-09-16T19:26:13Z" message: 'Readiness probe failed: Get "https://10.132.0.27:8643/healthz": dial tcp 10.132.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:26:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:26:13Z" name: error-404-isvc-4a90a-predictor-6759885b-smchc.18d5e3b068125a97 namespace: kserve-ci-e2e-test resourceVersion: "30572" uid: 07350fef-c97a-4a14-842c-25e416e1cbac reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-4a90a-predictor-6759885b namespace: kserve-ci-e2e-test resourceVersion: "26449" uid: 792c1e29-c8c9-4e7b-bc72-a34bff69d9e9 kind: Event lastTimestamp: "2026-09-16T19:16:56Z" message: 'Created pod: error-404-isvc-4a90a-predictor-6759885b-smchc' metadata: creationTimestamp: "2026-09-16T19:16:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:16:56Z" name: error-404-isvc-4a90a-predictor-6759885b.18d5e32ea9a7bc5d namespace: kserve-ci-e2e-test resourceVersion: "26456" uid: b754b580-64fb-4fbf-8b90-25e3f7d1f430 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-4a90a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26447" uid: 03753af2-afc9-4195-b8d7-daca9adc64f1 kind: Event lastTimestamp: "2026-09-16T19:16:56Z" message: Scaled up replica set error-404-isvc-4a90a-predictor-6759885b from 0 to 1 metadata: creationTimestamp: "2026-09-16T19:16:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:16:56Z" name: error-404-isvc-4a90a-predictor.18d5e32ea7513f18 namespace: kserve-ci-e2e-test resourceVersion: "26451" uid: 4dc9ec76-7b6d-4eea-b4ad-952aa76f8147 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:17:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4a90a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26448" uid: 1a99350c-c702-46c5-afc0-b17fe138a6c9 kind: Event lastTimestamp: "2026-09-16T19:17:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:17:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:17:11Z" name: error-404-isvc-4a90a-predictor.18d5e33225a0aa3a namespace: kserve-ci-e2e-test resourceVersion: "26623" uid: 6c6072a3-40b9-406f-9826-f8fc4e2d833f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:17:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4a90a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26448" uid: 1a99350c-c702-46c5-afc0-b17fe138a6c9 kind: Event lastTimestamp: "2026-09-16T19:17:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:17:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:17:11Z" name: error-404-isvc-4a90a-predictor.18d5e33225e23866 namespace: kserve-ci-e2e-test resourceVersion: "26624" uid: f95e2b10-8645-4d28-b33d-ad9a66fddc27 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-16T19:17:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4a90a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26622" uid: 1a99350c-c702-46c5-afc0-b17fe138a6c9 kind: Event lastTimestamp: "2026-09-16T19:18:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:17:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:18:26Z" name: error-404-isvc-4a90a-predictor.18d5e335a49e570f namespace: kserve-ci-e2e-test resourceVersion: "27324" uid: 22c98cdd-b5da-4067-8581-6941895bd567 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-16T19:17:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4a90a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26622" uid: 1a99350c-c702-46c5-afc0-b17fe138a6c9 kind: Event lastTimestamp: "2026-09-16T19:18:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:17:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:18:26Z" name: error-404-isvc-4a90a-predictor.18d5e335a50bcf13 namespace: kserve-ci-e2e-test resourceVersion: "27325" uid: 7045f227-82ea-48c6-93b6-dd3d5399780f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-16T19:16:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4a90a namespace: kserve-ci-e2e-test resourceVersion: "26432" uid: ec9b8ca5-3eb4-4986-920a-410c778c7429 kind: Event lastTimestamp: "2026-09-16T19:17:43Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T19:16:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:17:43Z" name: error-404-isvc-4a90a.18d5e32ea74fddc5 namespace: kserve-ci-e2e-test resourceVersion: "26955" uid: c7a208d4-f03c-4f9b-b53b-b9382d0b95e2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:17:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4a90a namespace: kserve-ci-e2e-test resourceVersion: "26952" uid: ec9b8ca5-3eb4-4986-920a-410c778c7429 kind: Event lastTimestamp: "2026-09-16T19:17:43Z" message: InferenceService [error-404-isvc-4a90a] is Ready metadata: creationTimestamp: "2026-09-16T19:17:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:17:43Z" name: error-404-isvc-4a90a.18d5e339b01f2fee namespace: kserve-ci-e2e-test resourceVersion: "26954" uid: 399908c6-5f09-4ee5-8a45-b9f199c8ee61 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:06:31.181007Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a3448-predictor-d94964498-cw5sn namespace: kserve-ci-e2e-test resourceVersion: "20883" uid: b3bef913-3e11-4581-a276-0a0bb5be512d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-a3448-predictor-d94964498-cw5sn to ip-10-0-138-79.ec2.internal metadata: creationTimestamp: "2026-09-16T19:06:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:06:31Z" name: error-404-isvc-a3448-predictor-d94964498-cw5sn.18d5e29d11806877 namespace: kserve-ci-e2e-test resourceVersion: "20889" uid: 8212d71a-3ab0-444e-8e43-0844d7b57245 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-594cc48779-9d5dg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:31Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a3448-predictor-d94964498-cw5sn namespace: kserve-ci-e2e-test resourceVersion: "20888" uid: b3bef913-3e11-4581-a276-0a0bb5be512d kind: Event lastTimestamp: "2026-09-16T19:06:31Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-a3448-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-16T19:06:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:06:31Z" name: error-404-isvc-a3448-predictor-d94964498-cw5sn.18d5e29d1f0a0fa8 namespace: kserve-ci-e2e-test resourceVersion: "20904" uid: 2830d489-c4ce-42e0-b215-2c0f5a513cec reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:32Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a3448-predictor-d94964498-cw5sn namespace: kserve-ci-e2e-test resourceVersion: "20895" uid: b3bef913-3e11-4581-a276-0a0bb5be512d kind: Event lastTimestamp: "2026-09-16T19:06:32Z" message: Add eth0 [10.132.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:06:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:06:32Z" name: error-404-isvc-a3448-predictor-d94964498-cw5sn.18d5e29d571df792 namespace: kserve-ci-e2e-test resourceVersion: "20925" uid: c1ad8339-4718-451f-9868-2c69f94b489c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a3448-predictor-d94964498-cw5sn namespace: kserve-ci-e2e-test resourceVersion: "20888" uid: b3bef913-3e11-4581-a276-0a0bb5be512d kind: Event lastTimestamp: "2026-09-16T19:06:32Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1986" already present on machine metadata: creationTimestamp: "2026-09-16T19:06:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:06:32Z" name: error-404-isvc-a3448-predictor-d94964498-cw5sn.18d5e29d58a61101 namespace: kserve-ci-e2e-test resourceVersion: "20927" uid: 1aad4adb-d618-4ee2-8bf0-1defed9ef48e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a3448-predictor-d94964498-cw5sn namespace: kserve-ci-e2e-test resourceVersion: "20888" uid: b3bef913-3e11-4581-a276-0a0bb5be512d kind: Event lastTimestamp: "2026-09-16T19:06:32Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T19:06:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:06:32Z" name: error-404-isvc-a3448-predictor-d94964498-cw5sn.18d5e29d5b8f3f4c namespace: kserve-ci-e2e-test resourceVersion: "20928" uid: 66ef8a38-f5d1-445d-9f90-222dd18e2f99 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a3448-predictor-d94964498-cw5sn namespace: kserve-ci-e2e-test resourceVersion: "20888" uid: b3bef913-3e11-4581-a276-0a0bb5be512d kind: Event lastTimestamp: "2026-09-16T19:06:32Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T19:06:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:06:32Z" name: error-404-isvc-a3448-predictor-d94964498-cw5sn.18d5e29d5c035d3b namespace: kserve-ci-e2e-test resourceVersion: "20929" uid: ff8db24e-7b41-46c4-a497-bb1a41e99b2b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a3448-predictor-d94964498-cw5sn namespace: kserve-ci-e2e-test resourceVersion: "20888" uid: b3bef913-3e11-4581-a276-0a0bb5be512d kind: Event lastTimestamp: "2026-09-16T19:06:32Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-16T19:06:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:06:32Z" name: error-404-isvc-a3448-predictor-d94964498-cw5sn.18d5e29d5c0d6590 namespace: kserve-ci-e2e-test resourceVersion: "20930" uid: 26e2055e-2cc9-4bf0-b850-803b5e5965c4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a3448-predictor-d94964498-cw5sn namespace: kserve-ci-e2e-test resourceVersion: "20888" uid: b3bef913-3e11-4581-a276-0a0bb5be512d kind: Event lastTimestamp: "2026-09-16T19:06:32Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T19:06:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:06:32Z" name: error-404-isvc-a3448-predictor-d94964498-cw5sn.18d5e29d5ec56bb1 namespace: kserve-ci-e2e-test resourceVersion: "20931" uid: b0449a3b-cabc-44f9-872f-2ba7cc9d515a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a3448-predictor-d94964498-cw5sn namespace: kserve-ci-e2e-test resourceVersion: "20888" uid: b3bef913-3e11-4581-a276-0a0bb5be512d kind: Event lastTimestamp: "2026-09-16T19:06:32Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:06:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:06:32Z" name: error-404-isvc-a3448-predictor-d94964498-cw5sn.18d5e29d5f5106d5 namespace: kserve-ci-e2e-test resourceVersion: "20932" uid: f626a628-832f-4c67-8039-e1ab562148b8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-16T19:06:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a3448-predictor-d94964498-cw5sn namespace: kserve-ci-e2e-test resourceVersion: "20888" uid: b3bef913-3e11-4581-a276-0a0bb5be512d kind: Event lastTimestamp: "2026-09-16T19:15:48Z" message: 'Readiness probe failed: dial tcp 10.132.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:06:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:15:48Z" name: error-404-isvc-a3448-predictor-d94964498-cw5sn.18d5e29d60abd749 namespace: kserve-ci-e2e-test resourceVersion: "25317" uid: ed3b4d17-97c0-4737-9154-f585714177a8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:15:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a3448-predictor-d94964498-cw5sn namespace: kserve-ci-e2e-test resourceVersion: "20888" uid: b3bef913-3e11-4581-a276-0a0bb5be512d kind: Event lastTimestamp: "2026-09-16T19:15:46Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T19:15:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:15:46Z" name: error-404-isvc-a3448-predictor-d94964498-cw5sn.18d5e31e466ede0b namespace: kserve-ci-e2e-test resourceVersion: "25241" uid: 46871b7d-40d3-4c14-8158-5c51d7a4041e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:15:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a3448-predictor-d94964498-cw5sn namespace: kserve-ci-e2e-test resourceVersion: "20888" uid: b3bef913-3e11-4581-a276-0a0bb5be512d kind: Event lastTimestamp: "2026-09-16T19:15:46Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:15:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:15:46Z" name: error-404-isvc-a3448-predictor-d94964498-cw5sn.18d5e31e466f5c71 namespace: kserve-ci-e2e-test resourceVersion: "25243" uid: 8829627b-97c2-4918-8141-61eb5f264182 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:15:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a3448-predictor-d94964498-cw5sn namespace: kserve-ci-e2e-test resourceVersion: "20888" uid: b3bef913-3e11-4581-a276-0a0bb5be512d kind: Event lastTimestamp: "2026-09-16T19:15:48Z" message: 'Readiness probe failed: Get "https://10.132.0.23:8643/healthz": dial tcp 10.132.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:15:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:15:48Z" name: error-404-isvc-a3448-predictor-d94964498-cw5sn.18d5e31ed50b727f namespace: kserve-ci-e2e-test resourceVersion: "25314" uid: 98e1bdfb-3f38-4061-97f8-34b4c98eb12d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-a3448-predictor-d94964498 namespace: kserve-ci-e2e-test resourceVersion: "20876" uid: b3e64189-f2b6-47ff-ad87-771d4ddcad32 kind: Event lastTimestamp: "2026-09-16T19:06:31Z" message: 'Created pod: error-404-isvc-a3448-predictor-d94964498-cw5sn' metadata: creationTimestamp: "2026-09-16T19:06:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:06:31Z" name: error-404-isvc-a3448-predictor-d94964498.18d5e29d10c38540 namespace: kserve-ci-e2e-test resourceVersion: "20886" uid: ca7a4dec-29c2-4a3d-b35a-25062629e154 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-a3448-predictor namespace: kserve-ci-e2e-test resourceVersion: "20874" uid: fa157452-6b74-4c93-af73-4480bd060ea3 kind: Event lastTimestamp: "2026-09-16T19:06:31Z" message: Scaled up replica set error-404-isvc-a3448-predictor-d94964498 from 0 to 1 metadata: creationTimestamp: "2026-09-16T19:06:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:06:31Z" name: error-404-isvc-a3448-predictor.18d5e29d0ecb49fe namespace: kserve-ci-e2e-test resourceVersion: "20878" uid: 7ea5d55a-13e8-4bf1-b3bc-ce9b51bb4568 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-16T19:06:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a3448-predictor namespace: kserve-ci-e2e-test resourceVersion: "20875" uid: 26a1901a-7cb8-4cf6-8cde-dbe1eb174774 kind: Event lastTimestamp: "2026-09-16T19:07:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:06:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:07:46Z" name: error-404-isvc-a3448-predictor.18d5e2a08d4630e4 namespace: kserve-ci-e2e-test resourceVersion: "21809" uid: 013deb89-8747-4015-8e18-f562c195f652 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-16T19:06:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a3448-predictor namespace: kserve-ci-e2e-test resourceVersion: "20875" uid: 26a1901a-7cb8-4cf6-8cde-dbe1eb174774 kind: Event lastTimestamp: "2026-09-16T19:07:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:06:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:07:46Z" name: error-404-isvc-a3448-predictor.18d5e2a08dbffe8e namespace: kserve-ci-e2e-test resourceVersion: "21810" uid: 2a22e61a-062b-4ce7-8468-87808e9e6083 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a3448-predictor namespace: kserve-ci-e2e-test resourceVersion: "21059" uid: 26a1901a-7cb8-4cf6-8cde-dbe1eb174774 kind: Event lastTimestamp: "2026-09-16T19:07:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:07:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:07:01Z" name: error-404-isvc-a3448-predictor.18d5e2a40c741b66 namespace: kserve-ci-e2e-test resourceVersion: "21236" uid: 9ea7d7ee-e994-4d4b-bcc9-929307b6a1c7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a3448-predictor namespace: kserve-ci-e2e-test resourceVersion: "21059" uid: 26a1901a-7cb8-4cf6-8cde-dbe1eb174774 kind: Event lastTimestamp: "2026-09-16T19:07:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:07:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:07:01Z" name: error-404-isvc-a3448-predictor.18d5e2a40ccb8d2f namespace: kserve-ci-e2e-test resourceVersion: "21238" uid: b3fc630b-ca0b-47cf-aab5-440b5c4823df reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T19:06:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a3448 namespace: kserve-ci-e2e-test resourceVersion: "20860" uid: 2e142b87-14c9-45b5-850c-33d984b235d2 kind: Event lastTimestamp: "2026-09-16T19:07:18Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T19:06:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:07:18Z" name: error-404-isvc-a3448.18d5e29d0ecd190e namespace: kserve-ci-e2e-test resourceVersion: "21545" uid: a4402582-788b-4ce4-964e-85dc38328e1f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a3448 namespace: kserve-ci-e2e-test resourceVersion: "21541" uid: 2e142b87-14c9-45b5-850c-33d984b235d2 kind: Event lastTimestamp: "2026-09-16T19:07:18Z" message: InferenceService [error-404-isvc-a3448] is Ready metadata: creationTimestamp: "2026-09-16T19:07:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:07:18Z" name: error-404-isvc-a3448.18d5e2a81e515f54 namespace: kserve-ci-e2e-test resourceVersion: "21542" uid: 1c7ba670-c56e-4817-a81b-b80ba8135510 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T18:54:25.898082Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-aa795-predictor-787c44f496-x4tph namespace: kserve-ci-e2e-test resourceVersion: "14484" uid: d4c0d47b-5238-4c06-8bcc-59b1d6374978 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-aa795-predictor-787c44f496-x4tph to ip-10-0-138-79.ec2.internal metadata: creationTimestamp: "2026-09-16T18:54:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T18:54:25Z" name: error-404-isvc-aa795-predictor-787c44f496-x4tph.18d5e1f43345472c namespace: kserve-ci-e2e-test resourceVersion: "14489" uid: 3717d855-0eae-493f-9278-7145ad07c4fe reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-594cc48779-9d5dg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:54:26Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-aa795-predictor-787c44f496-x4tph namespace: kserve-ci-e2e-test resourceVersion: "14494" uid: d4c0d47b-5238-4c06-8bcc-59b1d6374978 kind: Event lastTimestamp: "2026-09-16T18:54:26Z" message: Add eth0 [10.132.0.18/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T18:54:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T18:54:26Z" name: error-404-isvc-aa795-predictor-787c44f496-x4tph.18d5e1f455489281 namespace: kserve-ci-e2e-test resourceVersion: "14573" uid: 097e2233-9e84-4112-973f-0594934106ee reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:54:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-aa795-predictor-787c44f496-x4tph namespace: kserve-ci-e2e-test resourceVersion: "14486" uid: d4c0d47b-5238-4c06-8bcc-59b1d6374978 kind: Event lastTimestamp: "2026-09-16T18:54:26Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1986" metadata: creationTimestamp: "2026-09-16T18:54:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:54:26Z" name: error-404-isvc-aa795-predictor-787c44f496-x4tph.18d5e1f4569cc813 namespace: kserve-ci-e2e-test resourceVersion: "14576" uid: 9a2ce0f3-b14c-46ca-8591-840cb9a877a4 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:54:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-aa795-predictor-787c44f496-x4tph namespace: kserve-ci-e2e-test resourceVersion: "14486" uid: d4c0d47b-5238-4c06-8bcc-59b1d6374978 kind: Event lastTimestamp: "2026-09-16T18:54:38Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1986" in 11.714s (11.714s including waiting). Image size: 1355608146 bytes.' metadata: creationTimestamp: "2026-09-16T18:54:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:54:38Z" name: error-404-isvc-aa795-predictor-787c44f496-x4tph.18d5e1f710e1181b namespace: kserve-ci-e2e-test resourceVersion: "14702" uid: 4b78a3d2-a207-4946-a4a5-f61096d93db4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:54:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-aa795-predictor-787c44f496-x4tph namespace: kserve-ci-e2e-test resourceVersion: "14486" uid: d4c0d47b-5238-4c06-8bcc-59b1d6374978 kind: Event lastTimestamp: "2026-09-16T18:54:38Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T18:54:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:54:38Z" name: error-404-isvc-aa795-predictor-787c44f496-x4tph.18d5e1f7144b7a33 namespace: kserve-ci-e2e-test resourceVersion: "14703" uid: 7a540d0c-bc03-4c0e-8944-22a943ab7547 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:54:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-aa795-predictor-787c44f496-x4tph namespace: kserve-ci-e2e-test resourceVersion: "14486" uid: d4c0d47b-5238-4c06-8bcc-59b1d6374978 kind: Event lastTimestamp: "2026-09-16T18:54:38Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T18:54:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:54:38Z" name: error-404-isvc-aa795-predictor-787c44f496-x4tph.18d5e1f714d1e67c namespace: kserve-ci-e2e-test resourceVersion: "14704" uid: 699d11c9-9452-4904-9056-92dec49e9d8d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:54:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-aa795-predictor-787c44f496-x4tph namespace: kserve-ci-e2e-test resourceVersion: "14486" uid: d4c0d47b-5238-4c06-8bcc-59b1d6374978 kind: Event lastTimestamp: "2026-09-16T18:54:38Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-09-16T18:54:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:54:38Z" name: error-404-isvc-aa795-predictor-787c44f496-x4tph.18d5e1f714de869a namespace: kserve-ci-e2e-test resourceVersion: "14705" uid: e3b881dd-0b8f-4baf-893e-2e08b44b4d46 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:54:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-aa795-predictor-787c44f496-x4tph namespace: kserve-ci-e2e-test resourceVersion: "14486" uid: d4c0d47b-5238-4c06-8bcc-59b1d6374978 kind: Event lastTimestamp: "2026-09-16T18:54:40Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.25s (2.25s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-09-16T18:54:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:54:40Z" name: error-404-isvc-aa795-predictor-787c44f496-x4tph.18d5e1f79b0091b1 namespace: kserve-ci-e2e-test resourceVersion: "14724" uid: bb04e142-f1d9-4560-a464-9c9244d506b0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:54:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-aa795-predictor-787c44f496-x4tph namespace: kserve-ci-e2e-test resourceVersion: "14486" uid: d4c0d47b-5238-4c06-8bcc-59b1d6374978 kind: Event lastTimestamp: "2026-09-16T18:54:40Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T18:54:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:54:40Z" name: error-404-isvc-aa795-predictor-787c44f496-x4tph.18d5e1f79e1d2c60 namespace: kserve-ci-e2e-test resourceVersion: "14725" uid: bb72a60b-f2ec-4b12-887f-ea869957512e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:54:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-aa795-predictor-787c44f496-x4tph namespace: kserve-ci-e2e-test resourceVersion: "14486" uid: d4c0d47b-5238-4c06-8bcc-59b1d6374978 kind: Event lastTimestamp: "2026-09-16T18:54:40Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T18:54:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:54:40Z" name: error-404-isvc-aa795-predictor-787c44f496-x4tph.18d5e1f79e952dce namespace: kserve-ci-e2e-test resourceVersion: "14726" uid: ffb790a9-6d95-4888-b630-ebd8afff3193 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-16T18:54:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-aa795-predictor-787c44f496-x4tph namespace: kserve-ci-e2e-test resourceVersion: "14486" uid: d4c0d47b-5238-4c06-8bcc-59b1d6374978 kind: Event lastTimestamp: "2026-09-16T18:56:07Z" message: 'Readiness probe failed: dial tcp 10.132.0.18:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T18:54:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:07Z" name: error-404-isvc-aa795-predictor-787c44f496-x4tph.18d5e1f7b84cb7af namespace: kserve-ci-e2e-test resourceVersion: "15666" uid: 721eb2ae-4a4f-4177-9699-be254f6ff693 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-aa795-predictor-787c44f496-x4tph namespace: kserve-ci-e2e-test resourceVersion: "14486" uid: d4c0d47b-5238-4c06-8bcc-59b1d6374978 kind: Event lastTimestamp: "2026-09-16T18:56:05Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T18:56:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:05Z" name: error-404-isvc-aa795-predictor-787c44f496-x4tph.18d5e20b790fdc5b namespace: kserve-ci-e2e-test resourceVersion: "15571" uid: 7ad44cd8-f1e0-4cf1-93ba-07cc58ffe8e1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-aa795-predictor-787c44f496-x4tph namespace: kserve-ci-e2e-test resourceVersion: "14486" uid: d4c0d47b-5238-4c06-8bcc-59b1d6374978 kind: Event lastTimestamp: "2026-09-16T18:56:05Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T18:56:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:05Z" name: error-404-isvc-aa795-predictor-787c44f496-x4tph.18d5e20b79105f64 namespace: kserve-ci-e2e-test resourceVersion: "15572" uid: be3a17c3-4bd4-434b-a455-d99a61f6293a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-aa795-predictor-787c44f496-x4tph namespace: kserve-ci-e2e-test resourceVersion: "14486" uid: d4c0d47b-5238-4c06-8bcc-59b1d6374978 kind: Event lastTimestamp: "2026-09-16T18:56:07Z" message: 'Readiness probe failed: Get "https://10.132.0.18:8643/healthz": dial tcp 10.132.0.18:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-16T18:56:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:07Z" name: error-404-isvc-aa795-predictor-787c44f496-x4tph.18d5e20bbe7f2c7a namespace: kserve-ci-e2e-test resourceVersion: "15665" uid: 7a2324d9-f7ec-4971-9d64-7be446a01c44 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:54:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-aa795-predictor-787c44f496 namespace: kserve-ci-e2e-test resourceVersion: "14478" uid: 59d00828-c9a4-47cd-ac11-3d88bdfb9774 kind: Event lastTimestamp: "2026-09-16T18:54:25Z" message: 'Created pod: error-404-isvc-aa795-predictor-787c44f496-x4tph' metadata: creationTimestamp: "2026-09-16T18:54:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:54:25Z" name: error-404-isvc-aa795-predictor-787c44f496.18d5e1f432861f46 namespace: kserve-ci-e2e-test resourceVersion: "14485" uid: b5d491f8-cdb6-4a0a-a73f-a9913d46af12 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:54:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-aa795-predictor namespace: kserve-ci-e2e-test resourceVersion: "14475" uid: 20d5d400-2def-4b87-aa4c-3a1f58da90f8 kind: Event lastTimestamp: "2026-09-16T18:54:25Z" message: Scaled up replica set error-404-isvc-aa795-predictor-787c44f496 from 0 to 1 metadata: creationTimestamp: "2026-09-16T18:54:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:54:25Z" name: error-404-isvc-aa795-predictor.18d5e1f4305c0477 namespace: kserve-ci-e2e-test resourceVersion: "14479" uid: d575a4f8-def2-4dbd-9d6a-1e335edd6cd2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-16T18:54:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-aa795-predictor namespace: kserve-ci-e2e-test resourceVersion: "14477" uid: 022c8041-ff1c-4a02-8e55-b2c2b665c85a kind: Event lastTimestamp: "2026-09-16T18:55:25Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T18:54:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:55:25Z" name: error-404-isvc-aa795-predictor.18d5e1f7aee6bdfb namespace: kserve-ci-e2e-test resourceVersion: "15139" uid: 06e92487-c152-4674-ab21-054a90f64249 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-16T18:54:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-aa795-predictor namespace: kserve-ci-e2e-test resourceVersion: "14477" uid: 022c8041-ff1c-4a02-8e55-b2c2b665c85a kind: Event lastTimestamp: "2026-09-16T18:55:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T18:54:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:55:25Z" name: error-404-isvc-aa795-predictor.18d5e1f7af5026d3 namespace: kserve-ci-e2e-test resourceVersion: "15140" uid: 337255f2-5f23-4b77-9d3e-f372819b2162 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T18:55:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-aa795-predictor namespace: kserve-ci-e2e-test resourceVersion: "14736" uid: 022c8041-ff1c-4a02-8e55-b2c2b665c85a kind: Event lastTimestamp: "2026-09-16T18:55:55Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T18:55:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:55:55Z" name: error-404-isvc-aa795-predictor.18d5e205aa2cf496 namespace: kserve-ci-e2e-test resourceVersion: "15424" uid: 9e91aabb-21ff-45c4-ace7-326e9dfa5622 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T18:55:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-aa795-predictor namespace: kserve-ci-e2e-test resourceVersion: "14736" uid: 022c8041-ff1c-4a02-8e55-b2c2b665c85a kind: Event lastTimestamp: "2026-09-16T18:55:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T18:55:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:55:55Z" name: error-404-isvc-aa795-predictor.18d5e205aa960237 namespace: kserve-ci-e2e-test resourceVersion: "15425" uid: c74f8341-573f-45fd-b4c9-94adaaeda77d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-16T18:54:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-aa795 namespace: kserve-ci-e2e-test resourceVersion: "14459" uid: b0c79877-68ca-43b2-9262-cbb63c7189f8 kind: Event lastTimestamp: "2026-09-16T18:55:27Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T18:54:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T18:55:27Z" name: error-404-isvc-aa795.18d5e1f430627a89 namespace: kserve-ci-e2e-test resourceVersion: "15165" uid: a5b4639c-23dc-47e7-89cc-a334f04ad2d9 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:55:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-aa795 namespace: kserve-ci-e2e-test resourceVersion: "15163" uid: b0c79877-68ca-43b2-9262-cbb63c7189f8 kind: Event lastTimestamp: "2026-09-16T18:55:27Z" message: InferenceService [error-404-isvc-aa795] is Ready metadata: creationTimestamp: "2026-09-16T18:55:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T18:55:27Z" name: error-404-isvc-aa795.18d5e20273e0db66 namespace: kserve-ci-e2e-test resourceVersion: "15164" uid: 40c5e23f-3ae9-4788-82df-c9c51c9dbcd1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:05:21.022415Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c6de0-predictor-5ff88b4bf4-kdvms namespace: kserve-ci-e2e-test resourceVersion: "19991" uid: 81e01fdf-b5fe-46ce-b9e4-c515abdfdd2f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-c6de0-predictor-5ff88b4bf4-kdvms to ip-10-0-138-79.ec2.internal metadata: creationTimestamp: "2026-09-16T19:05:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:05:21Z" name: error-404-isvc-c6de0-predictor-5ff88b4bf4-kdvms.18d5e28cbbb94473 namespace: kserve-ci-e2e-test resourceVersion: "19995" uid: 8938f2c5-5d91-4724-b3c4-1c8bc2bd4b17 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-594cc48779-9d5dg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:05:21Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c6de0-predictor-5ff88b4bf4-kdvms namespace: kserve-ci-e2e-test resourceVersion: "19992" uid: 81e01fdf-b5fe-46ce-b9e4-c515abdfdd2f kind: Event lastTimestamp: "2026-09-16T19:05:21Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-c6de0-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-16T19:05:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:05:21Z" name: error-404-isvc-c6de0-predictor-5ff88b4bf4-kdvms.18d5e28ccb7fccb0 namespace: kserve-ci-e2e-test resourceVersion: "20006" uid: ca1a2bd1-fc57-450a-a6cc-633ea132b107 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:05:22Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c6de0-predictor-5ff88b4bf4-kdvms namespace: kserve-ci-e2e-test resourceVersion: "20001" uid: 81e01fdf-b5fe-46ce-b9e4-c515abdfdd2f kind: Event lastTimestamp: "2026-09-16T19:05:22Z" message: Add eth0 [10.132.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:05:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:05:22Z" name: error-404-isvc-c6de0-predictor-5ff88b4bf4-kdvms.18d5e28d0144152e namespace: kserve-ci-e2e-test resourceVersion: "20027" uid: 6cd030ce-1f39-4820-a9ca-0590f788f87f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:05:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c6de0-predictor-5ff88b4bf4-kdvms namespace: kserve-ci-e2e-test resourceVersion: "19992" uid: 81e01fdf-b5fe-46ce-b9e4-c515abdfdd2f kind: Event lastTimestamp: "2026-09-16T19:05:22Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1986" already present on machine metadata: creationTimestamp: "2026-09-16T19:05:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:05:22Z" name: error-404-isvc-c6de0-predictor-5ff88b4bf4-kdvms.18d5e28d02cd6b26 namespace: kserve-ci-e2e-test resourceVersion: "20030" uid: 790293c8-ed6f-45ad-92c9-0aae0b7b18fe reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:05:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c6de0-predictor-5ff88b4bf4-kdvms namespace: kserve-ci-e2e-test resourceVersion: "19992" uid: 81e01fdf-b5fe-46ce-b9e4-c515abdfdd2f kind: Event lastTimestamp: "2026-09-16T19:05:22Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T19:05:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:05:22Z" name: error-404-isvc-c6de0-predictor-5ff88b4bf4-kdvms.18d5e28d05e704ce namespace: kserve-ci-e2e-test resourceVersion: "20031" uid: 85c6577e-1e1e-4c70-8ea0-611b9a621777 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:05:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c6de0-predictor-5ff88b4bf4-kdvms namespace: kserve-ci-e2e-test resourceVersion: "19992" uid: 81e01fdf-b5fe-46ce-b9e4-c515abdfdd2f kind: Event lastTimestamp: "2026-09-16T19:05:22Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T19:05:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:05:22Z" name: error-404-isvc-c6de0-predictor-5ff88b4bf4-kdvms.18d5e28d065df184 namespace: kserve-ci-e2e-test resourceVersion: "20032" uid: 611de889-8b69-4f62-a526-0e2ce22be01d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:05:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c6de0-predictor-5ff88b4bf4-kdvms namespace: kserve-ci-e2e-test resourceVersion: "19992" uid: 81e01fdf-b5fe-46ce-b9e4-c515abdfdd2f kind: Event lastTimestamp: "2026-09-16T19:05:22Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-16T19:05:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:05:22Z" name: error-404-isvc-c6de0-predictor-5ff88b4bf4-kdvms.18d5e28d06679609 namespace: kserve-ci-e2e-test resourceVersion: "20033" uid: d9864a40-963e-4670-99bf-feec21918bf2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:05:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c6de0-predictor-5ff88b4bf4-kdvms namespace: kserve-ci-e2e-test resourceVersion: "19992" uid: 81e01fdf-b5fe-46ce-b9e4-c515abdfdd2f kind: Event lastTimestamp: "2026-09-16T19:05:22Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T19:05:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:05:22Z" name: error-404-isvc-c6de0-predictor-5ff88b4bf4-kdvms.18d5e28d090acba9 namespace: kserve-ci-e2e-test resourceVersion: "20034" uid: 88be8f33-451c-4d0b-9902-219f65d6b916 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:05:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c6de0-predictor-5ff88b4bf4-kdvms namespace: kserve-ci-e2e-test resourceVersion: "19992" uid: 81e01fdf-b5fe-46ce-b9e4-c515abdfdd2f kind: Event lastTimestamp: "2026-09-16T19:05:22Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:05:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:05:22Z" name: error-404-isvc-c6de0-predictor-5ff88b4bf4-kdvms.18d5e28d097f7cd2 namespace: kserve-ci-e2e-test resourceVersion: "20035" uid: f821d216-2aa7-453d-a9c1-673e53b7cbf8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T19:05:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c6de0-predictor-5ff88b4bf4-kdvms namespace: kserve-ci-e2e-test resourceVersion: "19992" uid: 81e01fdf-b5fe-46ce-b9e4-c515abdfdd2f kind: Event lastTimestamp: "2026-09-16T19:06:00Z" message: 'Readiness probe failed: dial tcp 10.132.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:05:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:06:00Z" name: error-404-isvc-c6de0-predictor-5ff88b4bf4-kdvms.18d5e28d7cc5f29a namespace: kserve-ci-e2e-test resourceVersion: "20334" uid: 5633f400-65df-444f-aaa4-153f9ce61f73 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c6de0-predictor-5ff88b4bf4-kdvms namespace: kserve-ci-e2e-test resourceVersion: "19992" uid: 81e01fdf-b5fe-46ce-b9e4-c515abdfdd2f kind: Event lastTimestamp: "2026-09-16T19:06:31Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T19:06:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:06:31Z" name: error-404-isvc-c6de0-predictor-5ff88b4bf4-kdvms.18d5e29d078f9e9e namespace: kserve-ci-e2e-test resourceVersion: "20847" uid: 74dbddf4-f0a5-4dc1-9284-eed6e17afbf9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c6de0-predictor-5ff88b4bf4-kdvms namespace: kserve-ci-e2e-test resourceVersion: "19992" uid: 81e01fdf-b5fe-46ce-b9e4-c515abdfdd2f kind: Event lastTimestamp: "2026-09-16T19:06:31Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:06:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:06:31Z" name: error-404-isvc-c6de0-predictor-5ff88b4bf4-kdvms.18d5e29d07903049 namespace: kserve-ci-e2e-test resourceVersion: "20850" uid: 16508b73-fdc9-4613-9354-dd90b748e1be reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:05:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-c6de0-predictor-5ff88b4bf4 namespace: kserve-ci-e2e-test resourceVersion: "19977" uid: d6b055cc-169e-4041-a03b-fdce741aa8b9 kind: Event lastTimestamp: "2026-09-16T19:05:21Z" message: 'Created pod: error-404-isvc-c6de0-predictor-5ff88b4bf4-kdvms' metadata: creationTimestamp: "2026-09-16T19:05:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:05:21Z" name: error-404-isvc-c6de0-predictor-5ff88b4bf4.18d5e28cbb258fa1 namespace: kserve-ci-e2e-test resourceVersion: "19993" uid: f9b39ad3-662d-4f4c-bd58-ff983f64b18e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:05:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-c6de0-predictor namespace: kserve-ci-e2e-test resourceVersion: "19974" uid: 44b7b815-cfde-45ff-a0e2-b886e66c5f7b kind: Event lastTimestamp: "2026-09-16T19:05:20Z" message: Scaled up replica set error-404-isvc-c6de0-predictor-5ff88b4bf4 from 0 to 1 metadata: creationTimestamp: "2026-09-16T19:05:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:05:20Z" name: error-404-isvc-c6de0-predictor.18d5e28cb80682b1 namespace: kserve-ci-e2e-test resourceVersion: "19980" uid: 51c92364-dab7-4705-8184-0117c9ce1610 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:05:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c6de0-predictor namespace: kserve-ci-e2e-test resourceVersion: "19975" uid: f24e3ba6-d8cc-48ad-92e9-46107ac16d6e kind: Event lastTimestamp: "2026-09-16T19:05:50Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:05:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:05:50Z" name: error-404-isvc-c6de0-predictor.18d5e2903661e873 namespace: kserve-ci-e2e-test resourceVersion: "20265" uid: 58db8e3f-547b-4457-ae47-2e753151dc2f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:05:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c6de0-predictor namespace: kserve-ci-e2e-test resourceVersion: "19975" uid: f24e3ba6-d8cc-48ad-92e9-46107ac16d6e kind: Event lastTimestamp: "2026-09-16T19:05:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:05:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:05:50Z" name: error-404-isvc-c6de0-predictor.18d5e29036d2b507 namespace: kserve-ci-e2e-test resourceVersion: "20266" uid: 61cadebd-49d9-4209-a7a3-5eb2dd84831d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:06:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c6de0-predictor namespace: kserve-ci-e2e-test resourceVersion: "20154" uid: f24e3ba6-d8cc-48ad-92e9-46107ac16d6e kind: Event lastTimestamp: "2026-09-16T19:06:21Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:06:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:06:21Z" name: error-404-isvc-c6de0-predictor.18d5e29734590054 namespace: kserve-ci-e2e-test resourceVersion: "20693" uid: 0033d326-83ff-4ba5-9e43-b07ef9aa9444 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:06:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c6de0-predictor namespace: kserve-ci-e2e-test resourceVersion: "20154" uid: f24e3ba6-d8cc-48ad-92e9-46107ac16d6e kind: Event lastTimestamp: "2026-09-16T19:06:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:06:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:06:21Z" name: error-404-isvc-c6de0-predictor.18d5e29734c62c64 namespace: kserve-ci-e2e-test resourceVersion: "20694" uid: b25e0755-e819-4512-b5f2-915e6002ee73 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T19:05:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c6de0 namespace: kserve-ci-e2e-test resourceVersion: "19944" uid: 3900d619-a294-427a-b6d0-791b58ec6867 kind: Event lastTimestamp: "2026-09-16T19:06:10Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T19:05:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:06:10Z" name: error-404-isvc-c6de0.18d5e28cb7fb2f44 namespace: kserve-ci-e2e-test resourceVersion: "20582" uid: 8f3e0e7d-2f25-443d-9874-6e16608cdc01 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:05:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c6de0 namespace: kserve-ci-e2e-test resourceVersion: "19944" uid: 3900d619-a294-427a-b6d0-791b58ec6867 kind: Event lastTimestamp: "2026-09-16T19:05:20Z" message: 'Failed to update status for InferenceService "error-404-isvc-c6de0": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-c6de0": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-16T19:05:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:05:20Z" name: error-404-isvc-c6de0.18d5e28cb8ca4a51 namespace: kserve-ci-e2e-test resourceVersion: "19986" uid: 8c3f57d0-7ed2-4962-8193-2424f566f295 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:05:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c6de0 namespace: kserve-ci-e2e-test resourceVersion: "19944" uid: 3900d619-a294-427a-b6d0-791b58ec6867 kind: Event lastTimestamp: "2026-09-16T19:05:20Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-c6de0": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-16T19:05:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:05:20Z" name: error-404-isvc-c6de0.18d5e28cb8ca9432 namespace: kserve-ci-e2e-test resourceVersion: "19987" uid: 7fb0a5cb-9020-4cda-87bc-d7a2b1e9fb5d reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c6de0 namespace: kserve-ci-e2e-test resourceVersion: "20580" uid: 3900d619-a294-427a-b6d0-791b58ec6867 kind: Event lastTimestamp: "2026-09-16T19:06:10Z" message: InferenceService [error-404-isvc-c6de0] is Ready metadata: creationTimestamp: "2026-09-16T19:06:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:06:10Z" name: error-404-isvc-c6de0.18d5e29837eb1141 namespace: kserve-ci-e2e-test resourceVersion: "20581" uid: 06af73aa-48b6-4cd4-b621-d4d04f4c75b5 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:16:26.044337Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ca22e-predictor-6b8dc9599b-mjzpn namespace: kserve-ci-e2e-test resourceVersion: "25710" uid: 655b1db8-2a90-41d2-9723-a2d8ed979c8c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-ca22e-predictor-6b8dc9599b-mjzpn to ip-10-0-138-79.ec2.internal metadata: creationTimestamp: "2026-09-16T19:16:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:16:26Z" name: error-404-isvc-ca22e-predictor-6b8dc9599b-mjzpn.18d5e327921e96b4 namespace: kserve-ci-e2e-test resourceVersion: "25716" uid: a878b6b9-097a-4257-b70e-7b91b0e97270 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-594cc48779-9d5dg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:26Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ca22e-predictor-6b8dc9599b-mjzpn namespace: kserve-ci-e2e-test resourceVersion: "25713" uid: 655b1db8-2a90-41d2-9723-a2d8ed979c8c kind: Event lastTimestamp: "2026-09-16T19:16:26Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-ca22e-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-16T19:16:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:16:26Z" name: error-404-isvc-ca22e-predictor-6b8dc9599b-mjzpn.18d5e3279e872ca8 namespace: kserve-ci-e2e-test resourceVersion: "25730" uid: 35d43b7d-991a-440b-b571-b65dab1da933 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:27Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ca22e-predictor-6b8dc9599b-mjzpn namespace: kserve-ci-e2e-test resourceVersion: "25724" uid: 655b1db8-2a90-41d2-9723-a2d8ed979c8c kind: Event lastTimestamp: "2026-09-16T19:16:27Z" message: Add eth0 [10.132.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:16:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:16:27Z" name: error-404-isvc-ca22e-predictor-6b8dc9599b-mjzpn.18d5e327d81471e4 namespace: kserve-ci-e2e-test resourceVersion: "25747" uid: 193df794-10d5-44c1-b868-860fb3dc969e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ca22e-predictor-6b8dc9599b-mjzpn namespace: kserve-ci-e2e-test resourceVersion: "25713" uid: 655b1db8-2a90-41d2-9723-a2d8ed979c8c kind: Event lastTimestamp: "2026-09-16T19:16:27Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1986" already present on machine metadata: creationTimestamp: "2026-09-16T19:16:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:16:27Z" name: error-404-isvc-ca22e-predictor-6b8dc9599b-mjzpn.18d5e327d9b3821e namespace: kserve-ci-e2e-test resourceVersion: "25749" uid: b4c0601b-db46-41c2-a682-3fc1cd6785cb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ca22e-predictor-6b8dc9599b-mjzpn namespace: kserve-ci-e2e-test resourceVersion: "25713" uid: 655b1db8-2a90-41d2-9723-a2d8ed979c8c kind: Event lastTimestamp: "2026-09-16T19:16:27Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T19:16:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:16:27Z" name: error-404-isvc-ca22e-predictor-6b8dc9599b-mjzpn.18d5e327dcf36640 namespace: kserve-ci-e2e-test resourceVersion: "25753" uid: ae3b1822-828e-48f2-bbcc-9d5efee7a3b8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ca22e-predictor-6b8dc9599b-mjzpn namespace: kserve-ci-e2e-test resourceVersion: "25713" uid: 655b1db8-2a90-41d2-9723-a2d8ed979c8c kind: Event lastTimestamp: "2026-09-16T19:16:27Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T19:16:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:16:27Z" name: error-404-isvc-ca22e-predictor-6b8dc9599b-mjzpn.18d5e327dd6907cd namespace: kserve-ci-e2e-test resourceVersion: "25754" uid: d4f0935b-5721-46b4-8bd9-d132eb2f10f6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ca22e-predictor-6b8dc9599b-mjzpn namespace: kserve-ci-e2e-test resourceVersion: "25713" uid: 655b1db8-2a90-41d2-9723-a2d8ed979c8c kind: Event lastTimestamp: "2026-09-16T19:16:27Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-16T19:16:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:16:27Z" name: error-404-isvc-ca22e-predictor-6b8dc9599b-mjzpn.18d5e327dd732e31 namespace: kserve-ci-e2e-test resourceVersion: "25755" uid: 73cadb40-fbba-4742-8665-0ee108864e5a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ca22e-predictor-6b8dc9599b-mjzpn namespace: kserve-ci-e2e-test resourceVersion: "25713" uid: 655b1db8-2a90-41d2-9723-a2d8ed979c8c kind: Event lastTimestamp: "2026-09-16T19:16:27Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T19:16:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:16:27Z" name: error-404-isvc-ca22e-predictor-6b8dc9599b-mjzpn.18d5e327e0714f9a namespace: kserve-ci-e2e-test resourceVersion: "25756" uid: 6dc47f88-4436-494d-8371-dfd60076a852 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ca22e-predictor-6b8dc9599b-mjzpn namespace: kserve-ci-e2e-test resourceVersion: "25713" uid: 655b1db8-2a90-41d2-9723-a2d8ed979c8c kind: Event lastTimestamp: "2026-09-16T19:16:27Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:16:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:16:27Z" name: error-404-isvc-ca22e-predictor-6b8dc9599b-mjzpn.18d5e327e0e69115 namespace: kserve-ci-e2e-test resourceVersion: "25757" uid: 3cc6377c-f507-4ac2-a75a-448af5da84e4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T19:16:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ca22e-predictor-6b8dc9599b-mjzpn namespace: kserve-ci-e2e-test resourceVersion: "25713" uid: 655b1db8-2a90-41d2-9723-a2d8ed979c8c kind: Event lastTimestamp: "2026-09-16T19:17:04Z" message: 'Readiness probe failed: dial tcp 10.132.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:16:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:17:04Z" name: error-404-isvc-ca22e-predictor-6b8dc9599b-mjzpn.18d5e3282c96d2b7 namespace: kserve-ci-e2e-test resourceVersion: "26565" uid: 334f53d2-8539-4375-94e9-c9f018fbcdfe reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:33:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ca22e-predictor-6b8dc9599b-mjzpn namespace: kserve-ci-e2e-test resourceVersion: "25713" uid: 655b1db8-2a90-41d2-9723-a2d8ed979c8c kind: Event lastTimestamp: "2026-09-16T19:33:45Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T19:33:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:33:45Z" name: error-404-isvc-ca22e-predictor-6b8dc9599b-mjzpn.18d5e4198e7eab2a namespace: kserve-ci-e2e-test resourceVersion: "33693" uid: 73d5ef12-b834-49d3-9686-624fd181d4b3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:33:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ca22e-predictor-6b8dc9599b-mjzpn namespace: kserve-ci-e2e-test resourceVersion: "25713" uid: 655b1db8-2a90-41d2-9723-a2d8ed979c8c kind: Event lastTimestamp: "2026-09-16T19:33:45Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:33:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:33:45Z" name: error-404-isvc-ca22e-predictor-6b8dc9599b-mjzpn.18d5e4198e7f3b9f namespace: kserve-ci-e2e-test resourceVersion: "33694" uid: a9cc3dcb-10c1-40d9-9ba3-8e114ba11f8a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-ca22e-predictor-6b8dc9599b namespace: kserve-ci-e2e-test resourceVersion: "25704" uid: d4327161-68e8-499c-ab34-19d6ad9a2632 kind: Event lastTimestamp: "2026-09-16T19:16:26Z" message: 'Created pod: error-404-isvc-ca22e-predictor-6b8dc9599b-mjzpn' metadata: creationTimestamp: "2026-09-16T19:16:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:16:26Z" name: error-404-isvc-ca22e-predictor-6b8dc9599b.18d5e32791801d80 namespace: kserve-ci-e2e-test resourceVersion: "25712" uid: 6b453bb1-6944-4bc8-b24c-2260baf15f04 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-ca22e-predictor namespace: kserve-ci-e2e-test resourceVersion: "25701" uid: d0aa0005-edfb-4a77-b87d-5c5e2a0ee9c0 kind: Event lastTimestamp: "2026-09-16T19:16:25Z" message: Scaled up replica set error-404-isvc-ca22e-predictor-6b8dc9599b from 0 to 1 metadata: creationTimestamp: "2026-09-16T19:16:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:16:25Z" name: error-404-isvc-ca22e-predictor.18d5e3278ea751f1 namespace: kserve-ci-e2e-test resourceVersion: "25706" uid: 80b3c752-b368-4ee9-a297-dbd80001ca7f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:16:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ca22e-predictor namespace: kserve-ci-e2e-test resourceVersion: "25703" uid: b7e6b3fb-736b-4211-8ed9-0dcc25631763 kind: Event lastTimestamp: "2026-09-16T19:17:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:16:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:17:11Z" name: error-404-isvc-ca22e-predictor.18d5e32b0dab8308 namespace: kserve-ci-e2e-test resourceVersion: "26613" uid: 543a7876-2f96-475d-92ae-f1c16205f787 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:16:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ca22e-predictor namespace: kserve-ci-e2e-test resourceVersion: "25703" uid: b7e6b3fb-736b-4211-8ed9-0dcc25631763 kind: Event lastTimestamp: "2026-09-16T19:17:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:16:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:17:11Z" name: error-404-isvc-ca22e-predictor.18d5e32b0e28cb9f namespace: kserve-ci-e2e-test resourceVersion: "26615" uid: e5b261a6-f43c-49e6-b098-84830d568588 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T19:16:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ca22e-predictor namespace: kserve-ci-e2e-test resourceVersion: "25905" uid: b7e6b3fb-736b-4211-8ed9-0dcc25631763 kind: Event lastTimestamp: "2026-09-16T19:17:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:16:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:17:41Z" name: error-404-isvc-ca22e-predictor.18d5e32e8cd72860 namespace: kserve-ci-e2e-test resourceVersion: "26914" uid: 5f712399-c1c7-4a57-b0e8-9b21ed25b1b4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T19:16:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ca22e-predictor namespace: kserve-ci-e2e-test resourceVersion: "25905" uid: b7e6b3fb-736b-4211-8ed9-0dcc25631763 kind: Event lastTimestamp: "2026-09-16T19:17:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:16:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:17:41Z" name: error-404-isvc-ca22e-predictor.18d5e32e8d49d35b namespace: kserve-ci-e2e-test resourceVersion: "26915" uid: 3c745f13-33bf-4b82-a9fe-12522ce367f6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T19:16:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ca22e namespace: kserve-ci-e2e-test resourceVersion: "25686" uid: cb49bd87-3cd8-4bcc-86a8-1626d8d814a3 kind: Event lastTimestamp: "2026-09-16T19:17:14Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T19:16:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:17:14Z" name: error-404-isvc-ca22e.18d5e3278ee724d2 namespace: kserve-ci-e2e-test resourceVersion: "26670" uid: 97f02a3c-39a0-4d2b-90ea-2170fbad56b9 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:17:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ca22e namespace: kserve-ci-e2e-test resourceVersion: "26667" uid: cb49bd87-3cd8-4bcc-86a8-1626d8d814a3 kind: Event lastTimestamp: "2026-09-16T19:17:14Z" message: InferenceService [error-404-isvc-ca22e] is Ready metadata: creationTimestamp: "2026-09-16T19:17:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:17:14Z" name: error-404-isvc-ca22e.18d5e332e84b2f7c namespace: kserve-ci-e2e-test resourceVersion: "26668" uid: 535cd467-aed8-4ffc-9a1e-032d0ca64f03 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:15:46.218415Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-da3eb-predictor-5c7778cd96-zctq4 namespace: kserve-ci-e2e-test resourceVersion: "25259" uid: 1a1c0844-99c2-4196-9069-33194ea2652e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-da3eb-predictor-5c7778cd96-zctq4 to ip-10-0-138-79.ec2.internal metadata: creationTimestamp: "2026-09-16T19:15:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:15:46Z" name: error-404-isvc-da3eb-predictor-5c7778cd96-zctq4.18d5e31e4c4f4074 namespace: kserve-ci-e2e-test resourceVersion: "25263" uid: 2a7362fe-0325-4b7d-82f4-46eb0c52b666 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-594cc48779-9d5dg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:15:46Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-da3eb-predictor-5c7778cd96-zctq4 namespace: kserve-ci-e2e-test resourceVersion: "25261" uid: 1a1c0844-99c2-4196-9069-33194ea2652e kind: Event lastTimestamp: "2026-09-16T19:15:46Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-da3eb-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-16T19:15:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:15:46Z" name: error-404-isvc-da3eb-predictor-5c7778cd96-zctq4.18d5e31e5b1d37fd namespace: kserve-ci-e2e-test resourceVersion: "25275" uid: 4608aa04-9fa9-443c-b1af-d733ae941e94 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:15:47Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-da3eb-predictor-5c7778cd96-zctq4 namespace: kserve-ci-e2e-test resourceVersion: "25268" uid: 1a1c0844-99c2-4196-9069-33194ea2652e kind: Event lastTimestamp: "2026-09-16T19:15:47Z" message: Add eth0 [10.132.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:15:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:15:47Z" name: error-404-isvc-da3eb-predictor-5c7778cd96-zctq4.18d5e31e91cf884c namespace: kserve-ci-e2e-test resourceVersion: "25297" uid: 2c0011a0-b029-4044-8128-359abda30d87 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:15:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-da3eb-predictor-5c7778cd96-zctq4 namespace: kserve-ci-e2e-test resourceVersion: "25261" uid: 1a1c0844-99c2-4196-9069-33194ea2652e kind: Event lastTimestamp: "2026-09-16T19:15:47Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1986" already present on machine metadata: creationTimestamp: "2026-09-16T19:15:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:15:47Z" name: error-404-isvc-da3eb-predictor-5c7778cd96-zctq4.18d5e31e94295c09 namespace: kserve-ci-e2e-test resourceVersion: "25300" uid: 43f1325e-a99a-40ea-a54a-0f50a2e2a635 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:15:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-da3eb-predictor-5c7778cd96-zctq4 namespace: kserve-ci-e2e-test resourceVersion: "25261" uid: 1a1c0844-99c2-4196-9069-33194ea2652e kind: Event lastTimestamp: "2026-09-16T19:15:47Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T19:15:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:15:47Z" name: error-404-isvc-da3eb-predictor-5c7778cd96-zctq4.18d5e31e972b1a2f namespace: kserve-ci-e2e-test resourceVersion: "25301" uid: ddd21625-9d8b-43e1-80a5-a24475e38022 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:15:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-da3eb-predictor-5c7778cd96-zctq4 namespace: kserve-ci-e2e-test resourceVersion: "25261" uid: 1a1c0844-99c2-4196-9069-33194ea2652e kind: Event lastTimestamp: "2026-09-16T19:15:47Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T19:15:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:15:47Z" name: error-404-isvc-da3eb-predictor-5c7778cd96-zctq4.18d5e31e97a5b108 namespace: kserve-ci-e2e-test resourceVersion: "25302" uid: af12639b-672b-41a1-882d-0a7144448082 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:15:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-da3eb-predictor-5c7778cd96-zctq4 namespace: kserve-ci-e2e-test resourceVersion: "25261" uid: 1a1c0844-99c2-4196-9069-33194ea2652e kind: Event lastTimestamp: "2026-09-16T19:15:47Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-16T19:15:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:15:47Z" name: error-404-isvc-da3eb-predictor-5c7778cd96-zctq4.18d5e31e97af3bce namespace: kserve-ci-e2e-test resourceVersion: "25303" uid: 29ba6732-9c77-4c75-b32d-cfd9c94529d6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:15:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-da3eb-predictor-5c7778cd96-zctq4 namespace: kserve-ci-e2e-test resourceVersion: "25261" uid: 1a1c0844-99c2-4196-9069-33194ea2652e kind: Event lastTimestamp: "2026-09-16T19:15:47Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T19:15:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:15:47Z" name: error-404-isvc-da3eb-predictor-5c7778cd96-zctq4.18d5e31e9a66ffda namespace: kserve-ci-e2e-test resourceVersion: "25304" uid: f25efaf2-3a4e-4ece-8dda-572be05b0980 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:15:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-da3eb-predictor-5c7778cd96-zctq4 namespace: kserve-ci-e2e-test resourceVersion: "25261" uid: 1a1c0844-99c2-4196-9069-33194ea2652e kind: Event lastTimestamp: "2026-09-16T19:15:47Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:15:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:15:47Z" name: error-404-isvc-da3eb-predictor-5c7778cd96-zctq4.18d5e31e9adccb6d namespace: kserve-ci-e2e-test resourceVersion: "25305" uid: 8a49b41a-1e02-4a55-bcc0-09137f9aadc7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T19:15:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-da3eb-predictor-5c7778cd96-zctq4 namespace: kserve-ci-e2e-test resourceVersion: "25261" uid: 1a1c0844-99c2-4196-9069-33194ea2652e kind: Event lastTimestamp: "2026-09-16T19:16:25Z" message: 'Readiness probe failed: dial tcp 10.132.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:15:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:16:25Z" name: error-404-isvc-da3eb-predictor-5c7778cd96-zctq4.18d5e31f0f215157 namespace: kserve-ci-e2e-test resourceVersion: "25613" uid: c717f3f1-ae1d-4875-b33b-bab478e543a3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-da3eb-predictor-5c7778cd96-zctq4 namespace: kserve-ci-e2e-test resourceVersion: "25261" uid: 1a1c0844-99c2-4196-9069-33194ea2652e kind: Event lastTimestamp: "2026-09-16T19:16:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T19:16:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:16:56Z" name: error-404-isvc-da3eb-predictor-5c7778cd96-zctq4.18d5e32e9c3f1923 namespace: kserve-ci-e2e-test resourceVersion: "26414" uid: 22d00dab-eb2e-40e3-bd3c-629bbdaed4e7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-da3eb-predictor-5c7778cd96-zctq4 namespace: kserve-ci-e2e-test resourceVersion: "25261" uid: 1a1c0844-99c2-4196-9069-33194ea2652e kind: Event lastTimestamp: "2026-09-16T19:16:56Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:16:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:16:56Z" name: error-404-isvc-da3eb-predictor-5c7778cd96-zctq4.18d5e32e9c3fdfb1 namespace: kserve-ci-e2e-test resourceVersion: "26415" uid: a437a79e-04fb-4193-a3a7-800ba4636dc4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:15:46Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-da3eb-predictor-5c7778cd96 namespace: kserve-ci-e2e-test resourceVersion: "25250" uid: 6557230c-cf42-4a4c-a63f-cef2aca0f4e3 kind: Event lastTimestamp: "2026-09-16T19:15:46Z" message: 'Created pod: error-404-isvc-da3eb-predictor-5c7778cd96-zctq4' metadata: creationTimestamp: "2026-09-16T19:15:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:15:46Z" name: error-404-isvc-da3eb-predictor-5c7778cd96.18d5e31e4bce0569 namespace: kserve-ci-e2e-test resourceVersion: "25260" uid: 68e3ae87-8d58-47b2-93c5-10b8d1a2c73c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:15:46Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-da3eb-predictor namespace: kserve-ci-e2e-test resourceVersion: "25247" uid: acb4b2e7-2438-44d7-95fe-f20bb6ae0b4d kind: Event lastTimestamp: "2026-09-16T19:15:46Z" message: Scaled up replica set error-404-isvc-da3eb-predictor-5c7778cd96 from 0 to 1 metadata: creationTimestamp: "2026-09-16T19:15:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:15:46Z" name: error-404-isvc-da3eb-predictor.18d5e31e49b903d5 namespace: kserve-ci-e2e-test resourceVersion: "25252" uid: fb3603de-1f36-4f46-806f-0fbbc0157a57 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T19:16:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-da3eb-predictor namespace: kserve-ci-e2e-test resourceVersion: "25249" uid: c3fdb472-3274-4368-a66d-45826321f5ad kind: Event lastTimestamp: "2026-09-16T19:16:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:16:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:16:31Z" name: error-404-isvc-da3eb-predictor.18d5e321c8889fb6 namespace: kserve-ci-e2e-test resourceVersion: "25802" uid: 4123dc62-09fa-42e1-b2d1-898d98dcf8a0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T19:16:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-da3eb-predictor namespace: kserve-ci-e2e-test resourceVersion: "25249" uid: c3fdb472-3274-4368-a66d-45826321f5ad kind: Event lastTimestamp: "2026-09-16T19:16:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:16:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:16:31Z" name: error-404-isvc-da3eb-predictor.18d5e321c910b2f1 namespace: kserve-ci-e2e-test resourceVersion: "25803" uid: 015e3440-5108-47ea-8894-614c302d1d11 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-da3eb-predictor namespace: kserve-ci-e2e-test resourceVersion: "25430" uid: c3fdb472-3274-4368-a66d-45826321f5ad kind: Event lastTimestamp: "2026-09-16T19:16:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:16:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:16:46Z" name: error-404-isvc-da3eb-predictor.18d5e32c45b03678 namespace: kserve-ci-e2e-test resourceVersion: "25977" uid: 4dba31d4-6438-4926-a6b9-d323ffbc1987 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-da3eb-predictor namespace: kserve-ci-e2e-test resourceVersion: "25430" uid: c3fdb472-3274-4368-a66d-45826321f5ad kind: Event lastTimestamp: "2026-09-16T19:16:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:16:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:16:46Z" name: error-404-isvc-da3eb-predictor.18d5e32c46236e15 namespace: kserve-ci-e2e-test resourceVersion: "25979" uid: 192bdf36-2c1b-41ea-a65b-f7e083b3a5eb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T19:15:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-da3eb namespace: kserve-ci-e2e-test resourceVersion: "25221" uid: 1001bf11-3e78-4ac5-ad31-a0c032c3a910 kind: Event lastTimestamp: "2026-09-16T19:16:35Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T19:15:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:16:35Z" name: error-404-isvc-da3eb.18d5e31e49f1d33b namespace: kserve-ci-e2e-test resourceVersion: "25865" uid: 02e69af6-d243-4113-8089-d6b02bc34121 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:15:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-da3eb namespace: kserve-ci-e2e-test resourceVersion: "25221" uid: 1001bf11-3e78-4ac5-ad31-a0c032c3a910 kind: Event lastTimestamp: "2026-09-16T19:15:46Z" message: 'Failed to update status for InferenceService "error-404-isvc-da3eb": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-da3eb": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-16T19:15:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:15:46Z" name: error-404-isvc-da3eb.18d5e31e4a83095a namespace: kserve-ci-e2e-test resourceVersion: "25255" uid: 0158dabb-ae00-4ee3-9916-219528f56799 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:15:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-da3eb namespace: kserve-ci-e2e-test resourceVersion: "25221" uid: 1001bf11-3e78-4ac5-ad31-a0c032c3a910 kind: Event lastTimestamp: "2026-09-16T19:15:46Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-da3eb": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-16T19:15:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:15:46Z" name: error-404-isvc-da3eb.18d5e31e4a833fb1 namespace: kserve-ci-e2e-test resourceVersion: "25256" uid: 2e119f1c-f14d-4194-a7e5-a6be5acf3cbd reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-da3eb namespace: kserve-ci-e2e-test resourceVersion: "25863" uid: 1001bf11-3e78-4ac5-ad31-a0c032c3a910 kind: Event lastTimestamp: "2026-09-16T19:16:35Z" message: InferenceService [error-404-isvc-da3eb] is Ready metadata: creationTimestamp: "2026-09-16T19:16:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:16:35Z" name: error-404-isvc-da3eb.18d5e329cba2fe5d namespace: kserve-ci-e2e-test resourceVersion: "25864" uid: 1657df02-7bfc-46ed-942c-80afe006ac17 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:07:11.126905Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e99c4-predictor-57c6bffb46-jrjhx namespace: kserve-ci-e2e-test resourceVersion: "21412" uid: 30fd4c11-16ff-431d-8dbb-6d01582305b8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-e99c4-predictor-57c6bffb46-jrjhx to ip-10-0-138-79.ec2.internal metadata: creationTimestamp: "2026-09-16T19:07:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:07:11Z" name: error-404-isvc-e99c4-predictor-57c6bffb46-jrjhx.18d5e2a65e76862f namespace: kserve-ci-e2e-test resourceVersion: "21416" uid: b451c830-2a96-4cc3-973d-9b1fef4752ed reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-594cc48779-9d5dg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:11Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e99c4-predictor-57c6bffb46-jrjhx namespace: kserve-ci-e2e-test resourceVersion: "21422" uid: 30fd4c11-16ff-431d-8dbb-6d01582305b8 kind: Event lastTimestamp: "2026-09-16T19:07:11Z" message: Add eth0 [10.132.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:07:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:07:11Z" name: error-404-isvc-e99c4-predictor-57c6bffb46-jrjhx.18d5e2a6804bb28c namespace: kserve-ci-e2e-test resourceVersion: "21437" uid: ec494a5d-30ea-4363-a14e-5d0d3d3e8c1c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e99c4-predictor-57c6bffb46-jrjhx namespace: kserve-ci-e2e-test resourceVersion: "21414" uid: 30fd4c11-16ff-431d-8dbb-6d01582305b8 kind: Event lastTimestamp: "2026-09-16T19:07:11Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1986" already present on machine metadata: creationTimestamp: "2026-09-16T19:07:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:11Z" name: error-404-isvc-e99c4-predictor-57c6bffb46-jrjhx.18d5e2a681ae400c namespace: kserve-ci-e2e-test resourceVersion: "21439" uid: a2a8032c-41d8-4327-a874-116d67194b13 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e99c4-predictor-57c6bffb46-jrjhx namespace: kserve-ci-e2e-test resourceVersion: "21414" uid: 30fd4c11-16ff-431d-8dbb-6d01582305b8 kind: Event lastTimestamp: "2026-09-16T19:07:11Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T19:07:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:11Z" name: error-404-isvc-e99c4-predictor-57c6bffb46-jrjhx.18d5e2a68445b559 namespace: kserve-ci-e2e-test resourceVersion: "21440" uid: fbf4bed9-387d-4831-aab4-70d87d06ceb5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e99c4-predictor-57c6bffb46-jrjhx namespace: kserve-ci-e2e-test resourceVersion: "21414" uid: 30fd4c11-16ff-431d-8dbb-6d01582305b8 kind: Event lastTimestamp: "2026-09-16T19:07:11Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T19:07:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:11Z" name: error-404-isvc-e99c4-predictor-57c6bffb46-jrjhx.18d5e2a684bd4479 namespace: kserve-ci-e2e-test resourceVersion: "21441" uid: 3a970623-4e80-4fb5-a116-3abf9a9de434 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e99c4-predictor-57c6bffb46-jrjhx namespace: kserve-ci-e2e-test resourceVersion: "21414" uid: 30fd4c11-16ff-431d-8dbb-6d01582305b8 kind: Event lastTimestamp: "2026-09-16T19:07:11Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-16T19:07:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:11Z" name: error-404-isvc-e99c4-predictor-57c6bffb46-jrjhx.18d5e2a684c6f03a namespace: kserve-ci-e2e-test resourceVersion: "21442" uid: cf1d1a29-02ee-4190-b108-0095ef629a91 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e99c4-predictor-57c6bffb46-jrjhx namespace: kserve-ci-e2e-test resourceVersion: "21414" uid: 30fd4c11-16ff-431d-8dbb-6d01582305b8 kind: Event lastTimestamp: "2026-09-16T19:07:11Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T19:07:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:11Z" name: error-404-isvc-e99c4-predictor-57c6bffb46-jrjhx.18d5e2a6873dd553 namespace: kserve-ci-e2e-test resourceVersion: "21444" uid: 5f641405-ebcf-43e4-9759-796560aac403 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e99c4-predictor-57c6bffb46-jrjhx namespace: kserve-ci-e2e-test resourceVersion: "21414" uid: 30fd4c11-16ff-431d-8dbb-6d01582305b8 kind: Event lastTimestamp: "2026-09-16T19:07:11Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:07:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:11Z" name: error-404-isvc-e99c4-predictor-57c6bffb46-jrjhx.18d5e2a687b5e757 namespace: kserve-ci-e2e-test resourceVersion: "21446" uid: 650bda3b-d247-4c07-bdc7-dda39803c6e8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-16T19:07:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e99c4-predictor-57c6bffb46-jrjhx namespace: kserve-ci-e2e-test resourceVersion: "21414" uid: 30fd4c11-16ff-431d-8dbb-6d01582305b8 kind: Event lastTimestamp: "2026-09-16T19:16:28Z" message: 'Readiness probe failed: dial tcp 10.132.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:07:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:16:28Z" name: error-404-isvc-e99c4-predictor-57c6bffb46-jrjhx.18d5e2a6ba3e1875 namespace: kserve-ci-e2e-test resourceVersion: "25774" uid: ccced01e-4f48-4382-83f8-6fe85ab16570 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e99c4-predictor-57c6bffb46-jrjhx namespace: kserve-ci-e2e-test resourceVersion: "21414" uid: 30fd4c11-16ff-431d-8dbb-6d01582305b8 kind: Event lastTimestamp: "2026-09-16T19:16:25Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T19:16:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:16:25Z" name: error-404-isvc-e99c4-predictor-57c6bffb46-jrjhx.18d5e3278092c6e5 namespace: kserve-ci-e2e-test resourceVersion: "25653" uid: ed315c25-2f77-43d0-8267-262bba920c0f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e99c4-predictor-57c6bffb46-jrjhx namespace: kserve-ci-e2e-test resourceVersion: "21414" uid: 30fd4c11-16ff-431d-8dbb-6d01582305b8 kind: Event lastTimestamp: "2026-09-16T19:16:25Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:16:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:16:25Z" name: error-404-isvc-e99c4-predictor-57c6bffb46-jrjhx.18d5e3278093e61b namespace: kserve-ci-e2e-test resourceVersion: "25658" uid: 3e2feb0e-0825-4a79-8a3a-b25842498a58 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e99c4-predictor-57c6bffb46-jrjhx namespace: kserve-ci-e2e-test resourceVersion: "21414" uid: 30fd4c11-16ff-431d-8dbb-6d01582305b8 kind: Event lastTimestamp: "2026-09-16T19:16:28Z" message: 'Readiness probe failed: Get "https://10.132.0.24:8643/healthz": dial tcp 10.132.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:16:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:16:28Z" name: error-404-isvc-e99c4-predictor-57c6bffb46-jrjhx.18d5e3282e8ed2a4 namespace: kserve-ci-e2e-test resourceVersion: "25773" uid: 882a71b1-8afa-4134-91b7-f7936d74ff16 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-e99c4-predictor-57c6bffb46 namespace: kserve-ci-e2e-test resourceVersion: "21404" uid: bb3fe859-7119-4778-a0d1-0fe564d648bc kind: Event lastTimestamp: "2026-09-16T19:07:11Z" message: 'Created pod: error-404-isvc-e99c4-predictor-57c6bffb46-jrjhx' metadata: creationTimestamp: "2026-09-16T19:07:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:07:11Z" name: error-404-isvc-e99c4-predictor-57c6bffb46.18d5e2a65e04e3f0 namespace: kserve-ci-e2e-test resourceVersion: "21413" uid: 460ccbc5-f303-4c87-ac74-10340d60643d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-e99c4-predictor namespace: kserve-ci-e2e-test resourceVersion: "21402" uid: 05a87f67-97f1-47bb-a1c3-3ed7ee98bc58 kind: Event lastTimestamp: "2026-09-16T19:07:11Z" message: Scaled up replica set error-404-isvc-e99c4-predictor-57c6bffb46 from 0 to 1 metadata: creationTimestamp: "2026-09-16T19:07:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:07:11Z" name: error-404-isvc-e99c4-predictor.18d5e2a65c32b4c3 namespace: kserve-ci-e2e-test resourceVersion: "21406" uid: d4ccb9f8-b607-461a-9dd6-3746ee46502f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:07:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e99c4-predictor namespace: kserve-ci-e2e-test resourceVersion: "21403" uid: a663c68a-2280-47c3-983e-8cdf74402502 kind: Event lastTimestamp: "2026-09-16T19:07:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:07:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:07:56Z" name: error-404-isvc-e99c4-predictor.18d5e2a9daaee1ae namespace: kserve-ci-e2e-test resourceVersion: "21880" uid: a9e80508-f073-46b1-a81d-dfc94d8c25af reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:07:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e99c4-predictor namespace: kserve-ci-e2e-test resourceVersion: "21403" uid: a663c68a-2280-47c3-983e-8cdf74402502 kind: Event lastTimestamp: "2026-09-16T19:07:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:07:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:07:56Z" name: error-404-isvc-e99c4-predictor.18d5e2a9db342a2c namespace: kserve-ci-e2e-test resourceVersion: "21881" uid: ae931290-73e2-4980-97ad-16b895f6c610 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-16T19:07:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e99c4-predictor namespace: kserve-ci-e2e-test resourceVersion: "21603" uid: a663c68a-2280-47c3-983e-8cdf74402502 kind: Event lastTimestamp: "2026-09-16T19:08:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:07:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:08:41Z" name: error-404-isvc-e99c4-predictor.18d5e2ad59ff20ce namespace: kserve-ci-e2e-test resourceVersion: "22268" uid: e22036d4-0e6d-4657-80fb-7847c2bfff42 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-16T19:07:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e99c4-predictor namespace: kserve-ci-e2e-test resourceVersion: "21603" uid: a663c68a-2280-47c3-983e-8cdf74402502 kind: Event lastTimestamp: "2026-09-16T19:08:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:07:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:08:41Z" name: error-404-isvc-e99c4-predictor.18d5e2ad5a68020f namespace: kserve-ci-e2e-test resourceVersion: "22269" uid: 46c38eb8-5c4b-4325-bbe2-fba65425c831 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-16T19:07:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e99c4 namespace: kserve-ci-e2e-test resourceVersion: "21392" uid: b8a037b0-30b5-43cf-af98-b7defbd48499 kind: Event lastTimestamp: "2026-09-16T19:07:58Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T19:07:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:07:58Z" name: error-404-isvc-e99c4.18d5e2a65c4daa5b namespace: kserve-ci-e2e-test resourceVersion: "21906" uid: 0829cfbb-36b8-4817-966d-3911583250ac reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e99c4 namespace: kserve-ci-e2e-test resourceVersion: "21392" uid: b8a037b0-30b5-43cf-af98-b7defbd48499 kind: Event lastTimestamp: "2026-09-16T19:07:11Z" message: 'Failed to update status for InferenceService "error-404-isvc-e99c4": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-e99c4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-16T19:07:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:07:11Z" name: error-404-isvc-e99c4.18d5e2a65cdbd0e7 namespace: kserve-ci-e2e-test resourceVersion: "21409" uid: 7ff875fa-e12d-4939-bf77-2b1afb5098b2 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e99c4 namespace: kserve-ci-e2e-test resourceVersion: "21392" uid: b8a037b0-30b5-43cf-af98-b7defbd48499 kind: Event lastTimestamp: "2026-09-16T19:07:11Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-e99c4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-16T19:07:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:07:11Z" name: error-404-isvc-e99c4.18d5e2a65cdc9f5e namespace: kserve-ci-e2e-test resourceVersion: "21410" uid: f1d13a9e-db79-462d-9439-006b8c7aac8c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e99c4 namespace: kserve-ci-e2e-test resourceVersion: "21903" uid: b8a037b0-30b5-43cf-af98-b7defbd48499 kind: Event lastTimestamp: "2026-09-16T19:07:58Z" message: InferenceService [error-404-isvc-e99c4] is Ready metadata: creationTimestamp: "2026-09-16T19:07:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:07:58Z" name: error-404-isvc-e99c4.18d5e2b1758b6395 namespace: kserve-ci-e2e-test resourceVersion: "21905" uid: f43d03e2-489a-4afc-8ad2-638b084cacfa reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T18:56:06.050919Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-eb09c-predictor-748fc5bc5b-xs866 namespace: kserve-ci-e2e-test resourceVersion: "15613" uid: 922928ff-8443-4b04-8bfd-174f8c9a58b0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-eb09c-predictor-748fc5bc5b-xs866 to ip-10-0-138-79.ec2.internal metadata: creationTimestamp: "2026-09-16T18:56:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T18:56:06Z" name: error-404-isvc-eb09c-predictor-748fc5bc5b-xs866.18d5e20b84d85aaf namespace: kserve-ci-e2e-test resourceVersion: "15618" uid: 11bf26ff-42a0-44e2-bc67-612defc17f98 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-594cc48779-9d5dg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:06Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-eb09c-predictor-748fc5bc5b-xs866 namespace: kserve-ci-e2e-test resourceVersion: "15622" uid: 922928ff-8443-4b04-8bfd-174f8c9a58b0 kind: Event lastTimestamp: "2026-09-16T18:56:06Z" message: Add eth0 [10.132.0.19/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T18:56:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T18:56:06Z" name: error-404-isvc-eb09c-predictor-748fc5bc5b-xs866.18d5e20ba6bde80b namespace: kserve-ci-e2e-test resourceVersion: "15648" uid: 01a65f98-5483-4025-a355-92d4761336c9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-eb09c-predictor-748fc5bc5b-xs866 namespace: kserve-ci-e2e-test resourceVersion: "15616" uid: 922928ff-8443-4b04-8bfd-174f8c9a58b0 kind: Event lastTimestamp: "2026-09-16T18:56:06Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1986" already present on machine metadata: creationTimestamp: "2026-09-16T18:56:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:06Z" name: error-404-isvc-eb09c-predictor-748fc5bc5b-xs866.18d5e20ba80a5e57 namespace: kserve-ci-e2e-test resourceVersion: "15652" uid: 37b4f3ba-3f23-4006-b51e-043a37e00658 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-eb09c-predictor-748fc5bc5b-xs866 namespace: kserve-ci-e2e-test resourceVersion: "15616" uid: 922928ff-8443-4b04-8bfd-174f8c9a58b0 kind: Event lastTimestamp: "2026-09-16T18:56:06Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T18:56:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:06Z" name: error-404-isvc-eb09c-predictor-748fc5bc5b-xs866.18d5e20baadf1eaa namespace: kserve-ci-e2e-test resourceVersion: "15655" uid: 56ad76fd-82e8-4fd1-b8ac-614b4d2567ca reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-eb09c-predictor-748fc5bc5b-xs866 namespace: kserve-ci-e2e-test resourceVersion: "15616" uid: 922928ff-8443-4b04-8bfd-174f8c9a58b0 kind: Event lastTimestamp: "2026-09-16T18:56:06Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T18:56:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:06Z" name: error-404-isvc-eb09c-predictor-748fc5bc5b-xs866.18d5e20bab5126d1 namespace: kserve-ci-e2e-test resourceVersion: "15656" uid: 59aa29de-d13e-4684-94bf-559b182089d7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-eb09c-predictor-748fc5bc5b-xs866 namespace: kserve-ci-e2e-test resourceVersion: "15616" uid: 922928ff-8443-4b04-8bfd-174f8c9a58b0 kind: Event lastTimestamp: "2026-09-16T18:56:06Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-16T18:56:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:06Z" name: error-404-isvc-eb09c-predictor-748fc5bc5b-xs866.18d5e20bab5bfc92 namespace: kserve-ci-e2e-test resourceVersion: "15657" uid: 29076688-92f7-49f0-8b52-8b723ed8c955 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-eb09c-predictor-748fc5bc5b-xs866 namespace: kserve-ci-e2e-test resourceVersion: "15616" uid: 922928ff-8443-4b04-8bfd-174f8c9a58b0 kind: Event lastTimestamp: "2026-09-16T18:56:06Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T18:56:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:06Z" name: error-404-isvc-eb09c-predictor-748fc5bc5b-xs866.18d5e20baed777c0 namespace: kserve-ci-e2e-test resourceVersion: "15658" uid: ab97c3f0-a0da-464f-85bb-3bf5f733219d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-eb09c-predictor-748fc5bc5b-xs866 namespace: kserve-ci-e2e-test resourceVersion: "15616" uid: 922928ff-8443-4b04-8bfd-174f8c9a58b0 kind: Event lastTimestamp: "2026-09-16T18:56:06Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T18:56:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:06Z" name: error-404-isvc-eb09c-predictor-748fc5bc5b-xs866.18d5e20baf4ba504 namespace: kserve-ci-e2e-test resourceVersion: "15659" uid: b2b46ca0-d2dd-4e67-b2e8-623337216af7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T18:56:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-eb09c-predictor-748fc5bc5b-xs866 namespace: kserve-ci-e2e-test resourceVersion: "15616" uid: 922928ff-8443-4b04-8bfd-174f8c9a58b0 kind: Event lastTimestamp: "2026-09-16T18:56:44Z" message: 'Readiness probe failed: dial tcp 10.132.0.19:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T18:56:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:44Z" name: error-404-isvc-eb09c-predictor-748fc5bc5b-xs866.18d5e20c0c53a114 namespace: kserve-ci-e2e-test resourceVersion: "16036" uid: a871490a-755e-4778-a466-a4eaca204e63 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:05:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-eb09c-predictor-748fc5bc5b-xs866 namespace: kserve-ci-e2e-test resourceVersion: "15616" uid: 922928ff-8443-4b04-8bfd-174f8c9a58b0 kind: Event lastTimestamp: "2026-09-16T19:05:20Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T19:05:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:05:20Z" name: error-404-isvc-eb09c-predictor-748fc5bc5b-xs866.18d5e28cb733ad33 namespace: kserve-ci-e2e-test resourceVersion: "19978" uid: 9fc118c5-5b3f-48cf-bec2-5fa4fcf940c9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:05:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-eb09c-predictor-748fc5bc5b-xs866 namespace: kserve-ci-e2e-test resourceVersion: "15616" uid: 922928ff-8443-4b04-8bfd-174f8c9a58b0 kind: Event lastTimestamp: "2026-09-16T19:05:20Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:05:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:05:20Z" name: error-404-isvc-eb09c-predictor-748fc5bc5b-xs866.18d5e28cb7344847 namespace: kserve-ci-e2e-test resourceVersion: "19981" uid: 5efdb631-6d26-43aa-af43-e8071f103a96 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-eb09c-predictor-748fc5bc5b namespace: kserve-ci-e2e-test resourceVersion: "15607" uid: 5da0df43-3584-4f8c-8ece-bb042de78edc kind: Event lastTimestamp: "2026-09-16T18:56:06Z" message: 'Created pod: error-404-isvc-eb09c-predictor-748fc5bc5b-xs866' metadata: creationTimestamp: "2026-09-16T18:56:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:56:06Z" name: error-404-isvc-eb09c-predictor-748fc5bc5b.18d5e20b8461ceac namespace: kserve-ci-e2e-test resourceVersion: "15615" uid: 197324b6-14ff-4c02-b04a-8f29255aa603 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-eb09c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15603" uid: 02f1a03f-9cf7-490e-b1fa-f1a8394c5912 kind: Event lastTimestamp: "2026-09-16T18:56:06Z" message: Scaled up replica set error-404-isvc-eb09c-predictor-748fc5bc5b from 0 to 1 metadata: creationTimestamp: "2026-09-16T18:56:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:56:06Z" name: error-404-isvc-eb09c-predictor.18d5e20b82ccfde6 namespace: kserve-ci-e2e-test resourceVersion: "15608" uid: 1f5e90ca-7ac6-47c9-bfe1-bcc371c79690 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T18:56:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-eb09c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15606" uid: 64f453a9-7b6a-47e5-93f7-9f7e51886c54 kind: Event lastTimestamp: "2026-09-16T18:56:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T18:56:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:56:36Z" name: error-404-isvc-eb09c-predictor.18d5e20f014af2e1 namespace: kserve-ci-e2e-test resourceVersion: "15956" uid: 08e19e1b-ad9c-4b83-a2f4-78d343dbe5bd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T18:56:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-eb09c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15606" uid: 64f453a9-7b6a-47e5-93f7-9f7e51886c54 kind: Event lastTimestamp: "2026-09-16T18:56:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T18:56:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:56:36Z" name: error-404-isvc-eb09c-predictor.18d5e20f01b635d6 namespace: kserve-ci-e2e-test resourceVersion: "15957" uid: 048b3202-40ac-4ff6-9eb8-e214163a7737 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T18:56:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-eb09c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15811" uid: 64f453a9-7b6a-47e5-93f7-9f7e51886c54 kind: Event lastTimestamp: "2026-09-16T18:57:21Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T18:56:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:57:21Z" name: error-404-isvc-eb09c-predictor.18d5e215ff25fbc4 namespace: kserve-ci-e2e-test resourceVersion: "16562" uid: 2824d4cc-69c8-42e8-8035-672aec48a2e5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T18:56:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-eb09c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15811" uid: 64f453a9-7b6a-47e5-93f7-9f7e51886c54 kind: Event lastTimestamp: "2026-09-16T18:57:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T18:56:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:57:21Z" name: error-404-isvc-eb09c-predictor.18d5e215ff94c876 namespace: kserve-ci-e2e-test resourceVersion: "16563" uid: bc543489-bd66-45de-9d37-d127a2aacf8c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-16T18:56:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-eb09c namespace: kserve-ci-e2e-test resourceVersion: "15587" uid: f8605dbb-1549-4582-9534-c87bafd5fbb3 kind: Event lastTimestamp: "2026-09-16T18:56:54Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T18:56:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T18:56:54Z" name: error-404-isvc-eb09c.18d5e20b82de14fe namespace: kserve-ci-e2e-test resourceVersion: "16303" uid: cd1b84db-c8f3-4201-a80e-c06fdb8967f8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-eb09c namespace: kserve-ci-e2e-test resourceVersion: "16301" uid: f8605dbb-1549-4582-9534-c87bafd5fbb3 kind: Event lastTimestamp: "2026-09-16T18:56:54Z" message: InferenceService [error-404-isvc-eb09c] is Ready metadata: creationTimestamp: "2026-09-16T18:56:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T18:56:54Z" name: error-404-isvc-eb09c.18d5e216c776074a namespace: kserve-ci-e2e-test resourceVersion: "16302" uid: 20811c63-aeee-4d6e-8412-fa8e982bc047 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:06:00.989344Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f64b9-predictor-56fdcf59dd-6cm9v namespace: kserve-ci-e2e-test resourceVersion: "20434" uid: b057a297-058b-4d65-a071-75f470208355 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-f64b9-predictor-56fdcf59dd-6cm9v to ip-10-0-138-79.ec2.internal metadata: creationTimestamp: "2026-09-16T19:06:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:06:00Z" name: error-404-isvc-f64b9-predictor-56fdcf59dd-6cm9v.18d5e29609f05362 namespace: kserve-ci-e2e-test resourceVersion: "20439" uid: 50c361d5-8e61-4dbd-afdb-030859acc9d6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-594cc48779-9d5dg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:01Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f64b9-predictor-56fdcf59dd-6cm9v namespace: kserve-ci-e2e-test resourceVersion: "20447" uid: b057a297-058b-4d65-a071-75f470208355 kind: Event lastTimestamp: "2026-09-16T19:06:01Z" message: Add eth0 [10.132.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:06:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:06:01Z" name: error-404-isvc-f64b9-predictor-56fdcf59dd-6cm9v.18d5e2962bf9150b namespace: kserve-ci-e2e-test resourceVersion: "20460" uid: d4c68e60-fb2a-46dc-94f3-b38b73cc0cb2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f64b9-predictor-56fdcf59dd-6cm9v namespace: kserve-ci-e2e-test resourceVersion: "20436" uid: b057a297-058b-4d65-a071-75f470208355 kind: Event lastTimestamp: "2026-09-16T19:06:01Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1986" already present on machine metadata: creationTimestamp: "2026-09-16T19:06:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:06:01Z" name: error-404-isvc-f64b9-predictor-56fdcf59dd-6cm9v.18d5e2962d6052ec namespace: kserve-ci-e2e-test resourceVersion: "20462" uid: 04136dc5-f3db-4d4a-a5a8-3f7fd7ac4c7a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f64b9-predictor-56fdcf59dd-6cm9v namespace: kserve-ci-e2e-test resourceVersion: "20436" uid: b057a297-058b-4d65-a071-75f470208355 kind: Event lastTimestamp: "2026-09-16T19:06:01Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T19:06:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:06:01Z" name: error-404-isvc-f64b9-predictor-56fdcf59dd-6cm9v.18d5e296301ef205 namespace: kserve-ci-e2e-test resourceVersion: "20463" uid: af212547-3c53-4f97-943e-1e4f82c860b9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f64b9-predictor-56fdcf59dd-6cm9v namespace: kserve-ci-e2e-test resourceVersion: "20436" uid: b057a297-058b-4d65-a071-75f470208355 kind: Event lastTimestamp: "2026-09-16T19:06:01Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T19:06:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:06:01Z" name: error-404-isvc-f64b9-predictor-56fdcf59dd-6cm9v.18d5e2963094c0e5 namespace: kserve-ci-e2e-test resourceVersion: "20464" uid: 3184a7cd-e232-4dc4-84f7-10126424c035 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f64b9-predictor-56fdcf59dd-6cm9v namespace: kserve-ci-e2e-test resourceVersion: "20436" uid: b057a297-058b-4d65-a071-75f470208355 kind: Event lastTimestamp: "2026-09-16T19:06:01Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-16T19:06:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:06:01Z" name: error-404-isvc-f64b9-predictor-56fdcf59dd-6cm9v.18d5e296309e2dc9 namespace: kserve-ci-e2e-test resourceVersion: "20465" uid: 327a6234-f532-4957-8180-4ef9986dd2e8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f64b9-predictor-56fdcf59dd-6cm9v namespace: kserve-ci-e2e-test resourceVersion: "20436" uid: b057a297-058b-4d65-a071-75f470208355 kind: Event lastTimestamp: "2026-09-16T19:06:01Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T19:06:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:06:01Z" name: error-404-isvc-f64b9-predictor-56fdcf59dd-6cm9v.18d5e29633e59ecd namespace: kserve-ci-e2e-test resourceVersion: "20466" uid: 41d47ef6-dc20-4d67-be3d-fcacf00b776b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f64b9-predictor-56fdcf59dd-6cm9v namespace: kserve-ci-e2e-test resourceVersion: "20436" uid: b057a297-058b-4d65-a071-75f470208355 kind: Event lastTimestamp: "2026-09-16T19:06:01Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:06:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:06:01Z" name: error-404-isvc-f64b9-predictor-56fdcf59dd-6cm9v.18d5e2963474ef88 namespace: kserve-ci-e2e-test resourceVersion: "20467" uid: 82b35b54-7d6b-4643-ba32-45a27a68f526 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T19:06:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f64b9-predictor-56fdcf59dd-6cm9v namespace: kserve-ci-e2e-test resourceVersion: "20436" uid: b057a297-058b-4d65-a071-75f470208355 kind: Event lastTimestamp: "2026-09-16T19:06:38Z" message: 'Readiness probe failed: dial tcp 10.132.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:06:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:06:38Z" name: error-404-isvc-f64b9-predictor-56fdcf59dd-6cm9v.18d5e2965dd1fd6b namespace: kserve-ci-e2e-test resourceVersion: "20992" uid: 02c4beda-13a1-4b4f-b867-b57bbf4e57c7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f64b9-predictor-56fdcf59dd-6cm9v namespace: kserve-ci-e2e-test resourceVersion: "20436" uid: b057a297-058b-4d65-a071-75f470208355 kind: Event lastTimestamp: "2026-09-16T19:07:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T19:07:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:10Z" name: error-404-isvc-f64b9-predictor-56fdcf59dd-6cm9v.18d5e2a655d49259 namespace: kserve-ci-e2e-test resourceVersion: "21378" uid: 2fd3bbde-6174-4f24-a87b-563ef4b09d61 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f64b9-predictor-56fdcf59dd-6cm9v namespace: kserve-ci-e2e-test resourceVersion: "20436" uid: b057a297-058b-4d65-a071-75f470208355 kind: Event lastTimestamp: "2026-09-16T19:07:10Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:07:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:10Z" name: error-404-isvc-f64b9-predictor-56fdcf59dd-6cm9v.18d5e2a655d51aea namespace: kserve-ci-e2e-test resourceVersion: "21382" uid: d7dddc13-7469-4af5-8b3b-3614c940f35d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f64b9-predictor-56fdcf59dd-6cm9v namespace: kserve-ci-e2e-test resourceVersion: "20436" uid: b057a297-058b-4d65-a071-75f470208355 kind: Event lastTimestamp: "2026-09-16T19:07:13Z" message: 'Readiness probe failed: Get "https://10.132.0.22:8643/healthz": dial tcp 10.132.0.22:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:07:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:13Z" name: error-404-isvc-f64b9-predictor-56fdcf59dd-6cm9v.18d5e2a6e5f092f6 namespace: kserve-ci-e2e-test resourceVersion: "21476" uid: 00379dd3-1a64-4459-919d-ebba612feeae reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-79.ec2.internal source: component: kubelet host: ip-10-0-138-79.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-f64b9-predictor-56fdcf59dd namespace: kserve-ci-e2e-test resourceVersion: "20426" uid: a8ad9a9f-7422-404f-bf5e-ca070e637f7b kind: Event lastTimestamp: "2026-09-16T19:06:00Z" message: 'Created pod: error-404-isvc-f64b9-predictor-56fdcf59dd-6cm9v' metadata: creationTimestamp: "2026-09-16T19:06:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:06:00Z" name: error-404-isvc-f64b9-predictor-56fdcf59dd.18d5e296095ef9ec namespace: kserve-ci-e2e-test resourceVersion: "20437" uid: ba4f466e-5023-424d-998f-87cd2473321f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-f64b9-predictor namespace: kserve-ci-e2e-test resourceVersion: "20425" uid: 7a02031c-4418-42f8-ad7a-95ea07262455 kind: Event lastTimestamp: "2026-09-16T19:06:00Z" message: Scaled up replica set error-404-isvc-f64b9-predictor-56fdcf59dd from 0 to 1 metadata: creationTimestamp: "2026-09-16T19:06:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:06:00Z" name: error-404-isvc-f64b9-predictor.18d5e296071d1a02 namespace: kserve-ci-e2e-test resourceVersion: "20428" uid: 5525fc01-7a27-43ec-8280-415e3b63dd11 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:06:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f64b9-predictor namespace: kserve-ci-e2e-test resourceVersion: "20427" uid: 95081902-e3ca-476a-8019-6d11de74114e kind: Event lastTimestamp: "2026-09-16T19:06:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:06:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:06:30Z" name: error-404-isvc-f64b9-predictor.18d5e29985f99926 namespace: kserve-ci-e2e-test resourceVersion: "20836" uid: 86ec0e39-93b3-4ed0-bd3c-a3791ca9e6ee reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:06:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f64b9-predictor namespace: kserve-ci-e2e-test resourceVersion: "20427" uid: 95081902-e3ca-476a-8019-6d11de74114e kind: Event lastTimestamp: "2026-09-16T19:06:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:06:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:06:30Z" name: error-404-isvc-f64b9-predictor.18d5e2998640381c namespace: kserve-ci-e2e-test resourceVersion: "20840" uid: fc01bea6-ce8e-45d4-abf3-a8a8d66f005c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:06:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f64b9-predictor namespace: kserve-ci-e2e-test resourceVersion: "20622" uid: 95081902-e3ca-476a-8019-6d11de74114e kind: Event lastTimestamp: "2026-09-16T19:07:01Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:06:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:07:01Z" name: error-404-isvc-f64b9-predictor.18d5e2a0843d9225 namespace: kserve-ci-e2e-test resourceVersion: "21227" uid: 348f8a58-b606-4027-8106-3660baa9f03e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:06:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f64b9-predictor namespace: kserve-ci-e2e-test resourceVersion: "20622" uid: 95081902-e3ca-476a-8019-6d11de74114e kind: Event lastTimestamp: "2026-09-16T19:07:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:06:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:07:01Z" name: error-404-isvc-f64b9-predictor.18d5e2a084a9d69c namespace: kserve-ci-e2e-test resourceVersion: "21229" uid: 36c3460d-b5b5-4ab4-a563-f18e2940b30a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-16T19:06:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f64b9 namespace: kserve-ci-e2e-test resourceVersion: "20413" uid: 2ae0b984-b876-4900-a136-83ab9dd3f3e9 kind: Event lastTimestamp: "2026-09-16T19:06:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T19:06:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:06:48Z" name: error-404-isvc-f64b9.18d5e29607771b6f namespace: kserve-ci-e2e-test resourceVersion: "21092" uid: 86935c20-399b-4270-aaaf-71ff81db145b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f64b9 namespace: kserve-ci-e2e-test resourceVersion: "20413" uid: 2ae0b984-b876-4900-a136-83ab9dd3f3e9 kind: Event lastTimestamp: "2026-09-16T19:06:00Z" message: 'Failed to update status for InferenceService "error-404-isvc-f64b9": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-f64b9": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-16T19:06:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:06:00Z" name: error-404-isvc-f64b9.18d5e29608292d79 namespace: kserve-ci-e2e-test resourceVersion: "20431" uid: 5c4c2366-3d33-494f-8586-aa19070fca07 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f64b9 namespace: kserve-ci-e2e-test resourceVersion: "20413" uid: 2ae0b984-b876-4900-a136-83ab9dd3f3e9 kind: Event lastTimestamp: "2026-09-16T19:06:00Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-f64b9": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-16T19:06:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:06:00Z" name: error-404-isvc-f64b9.18d5e29608298c9d namespace: kserve-ci-e2e-test resourceVersion: "20433" uid: cde76e77-0d83-4681-99d1-df675bc69309 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f64b9 namespace: kserve-ci-e2e-test resourceVersion: "21087" uid: 2ae0b984-b876-4900-a136-83ab9dd3f3e9 kind: Event lastTimestamp: "2026-09-16T19:06:48Z" message: InferenceService [error-404-isvc-f64b9] is Ready metadata: creationTimestamp: "2026-09-16T19:06:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:06:48Z" name: error-404-isvc-f64b9.18d5e2a11ae6a34b namespace: kserve-ci-e2e-test resourceVersion: "21089" uid: 87f7ee2f-bde1-45a3-ab0c-46ff36732a16 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T18:54:25.615994Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-595dc58456-t89gq namespace: kserve-ci-e2e-test resourceVersion: "14425" uid: 311b9459-414f-4afe-b06a-3e0e3f001570 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-595dc58456-t89gq to ip-10-0-130-26.ec2.internal metadata: creationTimestamp: "2026-09-16T18:54:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T18:54:25Z" name: isvc-sklearn-graph-1-predictor-595dc58456-t89gq.18d5e1f422753a92 namespace: kserve-ci-e2e-test resourceVersion: "14430" uid: 7a31d8a8-a022-4c60-bb23-ec1721725376 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-594cc48779-9d5dg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:54:26Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-595dc58456-t89gq namespace: kserve-ci-e2e-test resourceVersion: "14436" uid: 311b9459-414f-4afe-b06a-3e0e3f001570 kind: Event lastTimestamp: "2026-09-16T18:54:26Z" message: Add eth0 [10.134.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T18:54:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T18:54:26Z" name: isvc-sklearn-graph-1-predictor-595dc58456-t89gq.18d5e1f4451124e0 namespace: kserve-ci-e2e-test resourceVersion: "14549" uid: 27707064-5fa7-4b25-8fd3-8745b4a0bb49 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:54:26Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-595dc58456-t89gq namespace: kserve-ci-e2e-test resourceVersion: "14427" uid: 311b9459-414f-4afe-b06a-3e0e3f001570 kind: Event lastTimestamp: "2026-09-16T18:54:26Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:5e3f2c9028a91a0c41fb60e357bf2ecf428704c551e5155589c954076f13371d" already present on machine metadata: creationTimestamp: "2026-09-16T18:54:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:54:26Z" name: isvc-sklearn-graph-1-predictor-595dc58456-t89gq.18d5e1f446cb8609 namespace: kserve-ci-e2e-test resourceVersion: "14555" uid: ff675050-4f62-4076-8149-2ddf99fed021 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:54:26Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-595dc58456-t89gq namespace: kserve-ci-e2e-test resourceVersion: "14427" uid: 311b9459-414f-4afe-b06a-3e0e3f001570 kind: Event lastTimestamp: "2026-09-16T18:54:26Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-16T18:54:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:54:26Z" name: isvc-sklearn-graph-1-predictor-595dc58456-t89gq.18d5e1f44a2ebfa1 namespace: kserve-ci-e2e-test resourceVersion: "14559" uid: 0bc98f62-8de1-4783-85ee-ac0fb873a221 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:54:26Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-595dc58456-t89gq namespace: kserve-ci-e2e-test resourceVersion: "14427" uid: 311b9459-414f-4afe-b06a-3e0e3f001570 kind: Event lastTimestamp: "2026-09-16T18:54:26Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-16T18:54:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:54:26Z" name: isvc-sklearn-graph-1-predictor-595dc58456-t89gq.18d5e1f44aa39948 namespace: kserve-ci-e2e-test resourceVersion: "14560" uid: 8040fe39-04d2-4c73-b5f3-aa9dac55dcae reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:54:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-595dc58456-t89gq namespace: kserve-ci-e2e-test resourceVersion: "14427" uid: 311b9459-414f-4afe-b06a-3e0e3f001570 kind: Event lastTimestamp: "2026-09-16T18:54:31Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1986" metadata: creationTimestamp: "2026-09-16T18:54:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:54:31Z" name: isvc-sklearn-graph-1-predictor-595dc58456-t89gq.18d5e1f59e54b5be namespace: kserve-ci-e2e-test resourceVersion: "14641" uid: a4e6b6df-11e1-4abd-8e8f-e298e644f0db reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:54:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-595dc58456-t89gq namespace: kserve-ci-e2e-test resourceVersion: "14427" uid: 311b9459-414f-4afe-b06a-3e0e3f001570 kind: Event lastTimestamp: "2026-09-16T18:54:42Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1986" in 10.725s (10.725s including waiting). Image size: 1580859293 bytes.' metadata: creationTimestamp: "2026-09-16T18:54:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:54:42Z" name: isvc-sklearn-graph-1-predictor-595dc58456-t89gq.18d5e1f81da5ca20 namespace: kserve-ci-e2e-test resourceVersion: "14765" uid: b3411cf9-0732-494a-aa59-2dd29779e876 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:54:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-595dc58456-t89gq namespace: kserve-ci-e2e-test resourceVersion: "14427" uid: 311b9459-414f-4afe-b06a-3e0e3f001570 kind: Event lastTimestamp: "2026-09-16T18:54:42Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T18:54:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:54:42Z" name: isvc-sklearn-graph-1-predictor-595dc58456-t89gq.18d5e1f820c6c155 namespace: kserve-ci-e2e-test resourceVersion: "14769" uid: 40c63f9e-f2a1-4b25-9908-877552d1a368 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:54:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-595dc58456-t89gq namespace: kserve-ci-e2e-test resourceVersion: "14427" uid: 311b9459-414f-4afe-b06a-3e0e3f001570 kind: Event lastTimestamp: "2026-09-16T18:54:42Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T18:54:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:54:42Z" name: isvc-sklearn-graph-1-predictor-595dc58456-t89gq.18d5e1f8214120d6 namespace: kserve-ci-e2e-test resourceVersion: "14770" uid: 4b632246-e0b8-434a-8e5a-915c3311b55d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:54:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-595dc58456-t89gq namespace: kserve-ci-e2e-test resourceVersion: "14427" uid: 311b9459-414f-4afe-b06a-3e0e3f001570 kind: Event lastTimestamp: "2026-09-16T18:54:42Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-09-16T18:54:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:54:42Z" name: isvc-sklearn-graph-1-predictor-595dc58456-t89gq.18d5e1f8214f6cb8 namespace: kserve-ci-e2e-test resourceVersion: "14771" uid: 96243621-7712-47ec-8c21-beecc5fb4fe2 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:54:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-595dc58456-t89gq namespace: kserve-ci-e2e-test resourceVersion: "14427" uid: 311b9459-414f-4afe-b06a-3e0e3f001570 kind: Event lastTimestamp: "2026-09-16T18:54:47Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 4.278s (4.278s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-09-16T18:54:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:54:47Z" name: isvc-sklearn-graph-1-predictor-595dc58456-t89gq.18d5e1f920598fbb namespace: kserve-ci-e2e-test resourceVersion: "14807" uid: c6e8b764-248c-4ed6-aff1-5bde49878a58 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:54:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-595dc58456-t89gq namespace: kserve-ci-e2e-test resourceVersion: "14427" uid: 311b9459-414f-4afe-b06a-3e0e3f001570 kind: Event lastTimestamp: "2026-09-16T18:54:47Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T18:54:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:54:47Z" name: isvc-sklearn-graph-1-predictor-595dc58456-t89gq.18d5e1f924f16f3e namespace: kserve-ci-e2e-test resourceVersion: "14809" uid: 9ac93771-0241-4e53-9ea3-45074666f7d6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:54:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-595dc58456-t89gq namespace: kserve-ci-e2e-test resourceVersion: "14427" uid: 311b9459-414f-4afe-b06a-3e0e3f001570 kind: Event lastTimestamp: "2026-09-16T18:54:47Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T18:54:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:54:47Z" name: isvc-sklearn-graph-1-predictor-595dc58456-t89gq.18d5e1f925811f94 namespace: kserve-ci-e2e-test resourceVersion: "14812" uid: 9e3756d4-459c-4403-898b-5dc200008626 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-09-16T18:54:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-595dc58456-t89gq namespace: kserve-ci-e2e-test resourceVersion: "14427" uid: 311b9459-414f-4afe-b06a-3e0e3f001570 kind: Event lastTimestamp: "2026-09-16T18:56:47Z" message: 'Readiness probe failed: dial tcp 10.134.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T18:54:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:47Z" name: isvc-sklearn-graph-1-predictor-595dc58456-t89gq.18d5e1f95f6ad9d4 namespace: kserve-ci-e2e-test resourceVersion: "16208" uid: 2f8f6243-9052-4a0f-8d37-8362281df251 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-595dc58456-t89gq namespace: kserve-ci-e2e-test resourceVersion: "14427" uid: 311b9459-414f-4afe-b06a-3e0e3f001570 kind: Event lastTimestamp: "2026-09-16T18:56:45Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T18:56:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:45Z" name: isvc-sklearn-graph-1-predictor-595dc58456-t89gq.18d5e214c6a982a7 namespace: kserve-ci-e2e-test resourceVersion: "16108" uid: f0a84a30-7ce4-4bcf-bdcf-737eadadad6b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-595dc58456-t89gq namespace: kserve-ci-e2e-test resourceVersion: "14427" uid: 311b9459-414f-4afe-b06a-3e0e3f001570 kind: Event lastTimestamp: "2026-09-16T18:56:45Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T18:56:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:45Z" name: isvc-sklearn-graph-1-predictor-595dc58456-t89gq.18d5e214c6a9e595 namespace: kserve-ci-e2e-test resourceVersion: "16111" uid: 0a5dbc86-b1fa-4214-8a64-55c37755c256 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-595dc58456-t89gq namespace: kserve-ci-e2e-test resourceVersion: "14427" uid: 311b9459-414f-4afe-b06a-3e0e3f001570 kind: Event lastTimestamp: "2026-09-16T18:56:49Z" message: 'Readiness probe failed: Get "https://10.134.0.25:8643/healthz": dial tcp 10.134.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-16T18:56:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:49Z" name: isvc-sklearn-graph-1-predictor-595dc58456-t89gq.18d5e2158bfdec72 namespace: kserve-ci-e2e-test resourceVersion: "16231" uid: d705bff3-3d0b-4407-9f67-dab5c37c7772 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:54:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-595dc58456 namespace: kserve-ci-e2e-test resourceVersion: "14416" uid: 72ba35ba-0d24-47ff-90ac-b4847ade3f77 kind: Event lastTimestamp: "2026-09-16T18:54:25Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-595dc58456-t89gq' metadata: creationTimestamp: "2026-09-16T18:54:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:54:25Z" name: isvc-sklearn-graph-1-predictor-595dc58456.18d5e1f421fd55c8 namespace: kserve-ci-e2e-test resourceVersion: "14426" uid: 34d90441-fccc-47e1-a9a1-43660b680b9b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:54:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14415" uid: e6e6de82-1582-4e07-b3f8-e8fe49d132f6 kind: Event lastTimestamp: "2026-09-16T18:54:25Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-595dc58456 from 0 to 1 metadata: creationTimestamp: "2026-09-16T18:54:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:54:25Z" name: isvc-sklearn-graph-1-predictor.18d5e1f41d131ebf namespace: kserve-ci-e2e-test resourceVersion: "14417" uid: a396176b-e519-4a71-b9f3-c7686b5bc4b8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-16T18:54:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14429" uid: 02315ee8-05ff-4e6a-b217-89a4f0697f4b kind: Event lastTimestamp: "2026-09-16T18:55:40Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T18:54:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:55:40Z" name: isvc-sklearn-graph-1-predictor.18d5e1f7a28291f1 namespace: kserve-ci-e2e-test resourceVersion: "15267" uid: 157bf824-2ddf-4581-bdad-b403f2c8c9ca reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-16T18:54:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14429" uid: 02315ee8-05ff-4e6a-b217-89a4f0697f4b kind: Event lastTimestamp: "2026-09-16T18:55:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T18:54:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:55:40Z" name: isvc-sklearn-graph-1-predictor.18d5e1f7a2f50389 namespace: kserve-ci-e2e-test resourceVersion: "15268" uid: 18b131e4-dd6f-47c8-b96d-6c5223cea319 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-16T18:55:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14729" uid: 02315ee8-05ff-4e6a-b217-89a4f0697f4b kind: Event lastTimestamp: "2026-09-16T18:56:40Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T18:55:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:56:40Z" name: isvc-sklearn-graph-1-predictor.18d5e2091e45a6ca namespace: kserve-ci-e2e-test resourceVersion: "16002" uid: 6777e0b4-07df-4da9-870a-6725d6168b82 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-16T18:55:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14729" uid: 02315ee8-05ff-4e6a-b217-89a4f0697f4b kind: Event lastTimestamp: "2026-09-16T18:56:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T18:55:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:56:40Z" name: isvc-sklearn-graph-1-predictor.18d5e2091eab7594 namespace: kserve-ci-e2e-test resourceVersion: "16003" uid: 6b139435-2543-46e2-9d27-e40cffeb8401 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-09-16T18:54:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14404" uid: 6df43e11-657b-4176-95ff-fec9a7a30570 kind: Event lastTimestamp: "2026-09-16T18:56:07Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T18:54:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T18:56:07Z" name: isvc-sklearn-graph-1.18d5e1f4233cb990 namespace: kserve-ci-e2e-test resourceVersion: "15683" uid: 9c611b78-80f0-476d-be7a-bcc51e68a2bf reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "15681" uid: 6df43e11-657b-4176-95ff-fec9a7a30570 kind: Event lastTimestamp: "2026-09-16T18:56:07Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-09-16T18:56:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T18:56:07Z" name: isvc-sklearn-graph-1.18d5e20bde6529ce namespace: kserve-ci-e2e-test resourceVersion: "15682" uid: c9f9bd4b-772e-47f4-aa6f-9b0f1cd52e13 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T18:54:26.188392Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-6dfc766449-5r5nq namespace: kserve-ci-e2e-test resourceVersion: "14541" uid: f3a3a43a-76bc-4511-8c2e-db0346cbbaef kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-6dfc766449-5r5nq to ip-10-0-130-26.ec2.internal metadata: creationTimestamp: "2026-09-16T18:54:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T18:54:26Z" name: isvc-sklearn-graph-2-predictor-6dfc766449-5r5nq.18d5e1f44493217d namespace: kserve-ci-e2e-test resourceVersion: "14545" uid: 2caffef2-be23-4c3d-9443-bd8455d1bbae reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-594cc48779-9d5dg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:54:26Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-6dfc766449-5r5nq namespace: kserve-ci-e2e-test resourceVersion: "14553" uid: f3a3a43a-76bc-4511-8c2e-db0346cbbaef kind: Event lastTimestamp: "2026-09-16T18:54:26Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T18:54:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T18:54:26Z" name: isvc-sklearn-graph-2-predictor-6dfc766449-5r5nq.18d5e1f466f8badb namespace: kserve-ci-e2e-test resourceVersion: "14587" uid: f73f7da7-ca2d-4ceb-9a6a-6aed08e4514c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:54:26Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-6dfc766449-5r5nq namespace: kserve-ci-e2e-test resourceVersion: "14542" uid: f3a3a43a-76bc-4511-8c2e-db0346cbbaef kind: Event lastTimestamp: "2026-09-16T18:54:26Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:5e3f2c9028a91a0c41fb60e357bf2ecf428704c551e5155589c954076f13371d" already present on machine metadata: creationTimestamp: "2026-09-16T18:54:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:54:26Z" name: isvc-sklearn-graph-2-predictor-6dfc766449-5r5nq.18d5e1f4689dbfe4 namespace: kserve-ci-e2e-test resourceVersion: "14589" uid: 8da9aca3-5f79-4255-b253-d9427c6e3011 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:54:26Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-6dfc766449-5r5nq namespace: kserve-ci-e2e-test resourceVersion: "14542" uid: f3a3a43a-76bc-4511-8c2e-db0346cbbaef kind: Event lastTimestamp: "2026-09-16T18:54:26Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-16T18:54:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:54:26Z" name: isvc-sklearn-graph-2-predictor-6dfc766449-5r5nq.18d5e1f46dea4644 namespace: kserve-ci-e2e-test resourceVersion: "14590" uid: d0bd9544-86b3-47d4-813f-ba76e4499314 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:54:26Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-6dfc766449-5r5nq namespace: kserve-ci-e2e-test resourceVersion: "14542" uid: f3a3a43a-76bc-4511-8c2e-db0346cbbaef kind: Event lastTimestamp: "2026-09-16T18:54:26Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-16T18:54:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:54:26Z" name: isvc-sklearn-graph-2-predictor-6dfc766449-5r5nq.18d5e1f46e6f75b6 namespace: kserve-ci-e2e-test resourceVersion: "14591" uid: da00cec5-2ad2-466f-ada2-c037921d8d55 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:54:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6dfc766449-5r5nq namespace: kserve-ci-e2e-test resourceVersion: "14542" uid: f3a3a43a-76bc-4511-8c2e-db0346cbbaef kind: Event lastTimestamp: "2026-09-16T18:54:31Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1986" metadata: creationTimestamp: "2026-09-16T18:54:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:54:31Z" name: isvc-sklearn-graph-2-predictor-6dfc766449-5r5nq.18d5e1f59e368f0b namespace: kserve-ci-e2e-test resourceVersion: "14640" uid: 5df927c2-d189-422b-862b-b3aeb60db303 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:54:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6dfc766449-5r5nq namespace: kserve-ci-e2e-test resourceVersion: "14542" uid: f3a3a43a-76bc-4511-8c2e-db0346cbbaef kind: Event lastTimestamp: "2026-09-16T18:54:42Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1986" in 10.686s (10.686s including waiting). Image size: 1580859293 bytes.' metadata: creationTimestamp: "2026-09-16T18:54:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:54:42Z" name: isvc-sklearn-graph-2-predictor-6dfc766449-5r5nq.18d5e1f81b3359aa namespace: kserve-ci-e2e-test resourceVersion: "14763" uid: b89fa2e4-41ac-42d6-9bed-41d5ebdfebc1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:54:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6dfc766449-5r5nq namespace: kserve-ci-e2e-test resourceVersion: "14542" uid: f3a3a43a-76bc-4511-8c2e-db0346cbbaef kind: Event lastTimestamp: "2026-09-16T18:54:42Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T18:54:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:54:42Z" name: isvc-sklearn-graph-2-predictor-6dfc766449-5r5nq.18d5e1f81ef27a86 namespace: kserve-ci-e2e-test resourceVersion: "14766" uid: cfa68bf9-7713-4079-97a7-685fe9e484b3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:54:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6dfc766449-5r5nq namespace: kserve-ci-e2e-test resourceVersion: "14542" uid: f3a3a43a-76bc-4511-8c2e-db0346cbbaef kind: Event lastTimestamp: "2026-09-16T18:54:42Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T18:54:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:54:42Z" name: isvc-sklearn-graph-2-predictor-6dfc766449-5r5nq.18d5e1f81f701226 namespace: kserve-ci-e2e-test resourceVersion: "14767" uid: fa047ba2-e568-4171-a461-a514a437ff28 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:54:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6dfc766449-5r5nq namespace: kserve-ci-e2e-test resourceVersion: "14542" uid: f3a3a43a-76bc-4511-8c2e-db0346cbbaef kind: Event lastTimestamp: "2026-09-16T18:54:42Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-09-16T18:54:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:54:42Z" name: isvc-sklearn-graph-2-predictor-6dfc766449-5r5nq.18d5e1f81f7d7c86 namespace: kserve-ci-e2e-test resourceVersion: "14768" uid: 547eee17-83d3-41f4-9a19-fb6fa6122f95 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:54:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6dfc766449-5r5nq namespace: kserve-ci-e2e-test resourceVersion: "14542" uid: f3a3a43a-76bc-4511-8c2e-db0346cbbaef kind: Event lastTimestamp: "2026-09-16T18:54:47Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 4.276s (4.276s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-09-16T18:54:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:54:47Z" name: isvc-sklearn-graph-2-predictor-6dfc766449-5r5nq.18d5e1f91e62d92b namespace: kserve-ci-e2e-test resourceVersion: "14803" uid: c2b21a85-8ebd-4304-aad0-efec97e1619e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:54:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6dfc766449-5r5nq namespace: kserve-ci-e2e-test resourceVersion: "14542" uid: f3a3a43a-76bc-4511-8c2e-db0346cbbaef kind: Event lastTimestamp: "2026-09-16T18:54:47Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T18:54:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:54:47Z" name: isvc-sklearn-graph-2-predictor-6dfc766449-5r5nq.18d5e1f924fdfd47 namespace: kserve-ci-e2e-test resourceVersion: "14811" uid: d480a08e-33ba-44da-b763-46ac0d8024d6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:54:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6dfc766449-5r5nq namespace: kserve-ci-e2e-test resourceVersion: "14542" uid: f3a3a43a-76bc-4511-8c2e-db0346cbbaef kind: Event lastTimestamp: "2026-09-16T18:54:47Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T18:54:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:54:47Z" name: isvc-sklearn-graph-2-predictor-6dfc766449-5r5nq.18d5e1f925a95223 namespace: kserve-ci-e2e-test resourceVersion: "14814" uid: 71b41c3b-aabd-4ec2-80a4-80d2eb561a00 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-09-16T18:54:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6dfc766449-5r5nq namespace: kserve-ci-e2e-test resourceVersion: "14542" uid: f3a3a43a-76bc-4511-8c2e-db0346cbbaef kind: Event lastTimestamp: "2026-09-16T18:56:46Z" message: 'Readiness probe failed: dial tcp 10.134.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T18:54:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:46Z" name: isvc-sklearn-graph-2-predictor-6dfc766449-5r5nq.18d5e1f99b7981d7 namespace: kserve-ci-e2e-test resourceVersion: "16182" uid: 7c1fdb9f-a107-4471-b515-72704b792302 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6dfc766449-5r5nq namespace: kserve-ci-e2e-test resourceVersion: "14542" uid: f3a3a43a-76bc-4511-8c2e-db0346cbbaef kind: Event lastTimestamp: "2026-09-16T18:56:45Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T18:56:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:45Z" name: isvc-sklearn-graph-2-predictor-6dfc766449-5r5nq.18d5e214c3ea89cc namespace: kserve-ci-e2e-test resourceVersion: "16084" uid: 0e8d1096-b837-464b-b516-c8e7ae0b6996 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6dfc766449-5r5nq namespace: kserve-ci-e2e-test resourceVersion: "14542" uid: f3a3a43a-76bc-4511-8c2e-db0346cbbaef kind: Event lastTimestamp: "2026-09-16T18:56:45Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T18:56:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:45Z" name: isvc-sklearn-graph-2-predictor-6dfc766449-5r5nq.18d5e214c3eaee3a namespace: kserve-ci-e2e-test resourceVersion: "16089" uid: 8cf999d0-b4dc-4282-9f9e-df1081972f84 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6dfc766449-5r5nq namespace: kserve-ci-e2e-test resourceVersion: "14542" uid: f3a3a43a-76bc-4511-8c2e-db0346cbbaef kind: Event lastTimestamp: "2026-09-16T18:56:50Z" message: 'Readiness probe failed: Get "https://10.134.0.28:8643/healthz": dial tcp 10.134.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-16T18:56:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:50Z" name: isvc-sklearn-graph-2-predictor-6dfc766449-5r5nq.18d5e215c7d629ec namespace: kserve-ci-e2e-test resourceVersion: "16240" uid: 7732fa31-1967-42f5-b247-9e6645ec9515 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:54:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-6dfc766449 namespace: kserve-ci-e2e-test resourceVersion: "14534" uid: c469f654-0d99-4c4f-8f9b-6b245bc55153 kind: Event lastTimestamp: "2026-09-16T18:54:26Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-6dfc766449-5r5nq' metadata: creationTimestamp: "2026-09-16T18:54:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:54:26Z" name: isvc-sklearn-graph-2-predictor-6dfc766449.18d5e1f4442dbfeb namespace: kserve-ci-e2e-test resourceVersion: "14543" uid: ac06ff56-b795-49a7-ae09-91188d6f7568 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:54:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14529" uid: 3cc17fb9-2ea9-42f6-8fbb-ba7794cb99aa kind: Event lastTimestamp: "2026-09-16T18:54:26Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-6dfc766449 from 0 to 1 metadata: creationTimestamp: "2026-09-16T18:54:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:54:26Z" name: isvc-sklearn-graph-2-predictor.18d5e1f44130886a namespace: kserve-ci-e2e-test resourceVersion: "14535" uid: 6aac7012-daa0-425a-90fa-5a9a568ea131 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T18:54:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14533" uid: 8a2e2f3f-9417-4b94-ad44-f368e48dcb39 kind: Event lastTimestamp: "2026-09-16T18:55:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T18:54:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:55:26Z" name: isvc-sklearn-graph-2-predictor.18d5e1f7bfed58b7 namespace: kserve-ci-e2e-test resourceVersion: "15147" uid: f2061b56-c7ca-4f7e-972e-ec4a9108338f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T18:54:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14533" uid: 8a2e2f3f-9417-4b94-ad44-f368e48dcb39 kind: Event lastTimestamp: "2026-09-16T18:55:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T18:54:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:55:26Z" name: isvc-sklearn-graph-2-predictor.18d5e1f7c054312d namespace: kserve-ci-e2e-test resourceVersion: "15149" uid: 1f8c1ecf-0d4e-4acb-8768-18aee8d4c684 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T18:55:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14748" uid: 8a2e2f3f-9417-4b94-ad44-f368e48dcb39 kind: Event lastTimestamp: "2026-09-16T18:56:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T18:55:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:56:41Z" name: isvc-sklearn-graph-2-predictor.18d5e1febdb476f4 namespace: kserve-ci-e2e-test resourceVersion: "16009" uid: b98d3419-f723-44cb-8666-fb36188cce51 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T18:55:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14748" uid: 8a2e2f3f-9417-4b94-ad44-f368e48dcb39 kind: Event lastTimestamp: "2026-09-16T18:56:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T18:55:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:56:41Z" name: isvc-sklearn-graph-2-predictor.18d5e1febe1c243b namespace: kserve-ci-e2e-test resourceVersion: "16010" uid: 4205d0f7-463e-467e-841a-55c41b15ea93 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-09-16T18:54:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14515" uid: 2f44e2bb-51d7-4d0a-aa6e-825d99c680ff kind: Event lastTimestamp: "2026-09-16T18:56:06Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T18:54:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T18:56:06Z" name: isvc-sklearn-graph-2.18d5e1f44163918e namespace: kserve-ci-e2e-test resourceVersion: "15644" uid: 1aa38d0a-0326-4789-b773-1c964f02c88c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "15640" uid: 2f44e2bb-51d7-4d0a-aa6e-825d99c680ff kind: Event lastTimestamp: "2026-09-16T18:56:06Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-09-16T18:56:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T18:56:06Z" name: isvc-sklearn-graph-2.18d5e20ba2993bc8 namespace: kserve-ci-e2e-test resourceVersion: "15641" uid: ec2bc297-3386-45e5-83cc-e82186a1d37e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T18:54:26.086697Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-4pzzs namespace: kserve-ci-e2e-test resourceVersion: "14516" uid: 9c79e632-ba98-4a09-ad37-0992ed0e90f1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-555fc69d56-4pzzs to ip-10-0-130-26.ec2.internal metadata: creationTimestamp: "2026-09-16T18:54:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T18:54:26Z" name: isvc-xgboost-graph-predictor-555fc69d56-4pzzs.18d5e1f43e835f52 namespace: kserve-ci-e2e-test resourceVersion: "14520" uid: 1395ff54-daf9-4de0-867d-d3967036171e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-594cc48779-9d5dg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:54:26Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-4pzzs namespace: kserve-ci-e2e-test resourceVersion: "14531" uid: 9c79e632-ba98-4a09-ad37-0992ed0e90f1 kind: Event lastTimestamp: "2026-09-16T18:54:26Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T18:54:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T18:54:26Z" name: isvc-xgboost-graph-predictor-555fc69d56-4pzzs.18d5e1f460bcec74 namespace: kserve-ci-e2e-test resourceVersion: "14582" uid: 6dce0a3b-c161-404b-adf7-6765b2802b55 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:54:26Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-4pzzs namespace: kserve-ci-e2e-test resourceVersion: "14518" uid: 9c79e632-ba98-4a09-ad37-0992ed0e90f1 kind: Event lastTimestamp: "2026-09-16T18:54:26Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:5e3f2c9028a91a0c41fb60e357bf2ecf428704c551e5155589c954076f13371d" already present on machine metadata: creationTimestamp: "2026-09-16T18:54:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:54:26Z" name: isvc-xgboost-graph-predictor-555fc69d56-4pzzs.18d5e1f462415731 namespace: kserve-ci-e2e-test resourceVersion: "14584" uid: 31a902c6-e943-416a-a27d-e8402fe8c5e7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:54:26Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-4pzzs namespace: kserve-ci-e2e-test resourceVersion: "14518" uid: 9c79e632-ba98-4a09-ad37-0992ed0e90f1 kind: Event lastTimestamp: "2026-09-16T18:54:26Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-16T18:54:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:54:26Z" name: isvc-xgboost-graph-predictor-555fc69d56-4pzzs.18d5e1f465910f49 namespace: kserve-ci-e2e-test resourceVersion: "14585" uid: 2c5eee0d-dbbf-432a-9c53-3495258db763 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:54:26Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-4pzzs namespace: kserve-ci-e2e-test resourceVersion: "14518" uid: 9c79e632-ba98-4a09-ad37-0992ed0e90f1 kind: Event lastTimestamp: "2026-09-16T18:54:26Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-16T18:54:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:54:26Z" name: isvc-xgboost-graph-predictor-555fc69d56-4pzzs.18d5e1f46606737e namespace: kserve-ci-e2e-test resourceVersion: "14586" uid: 4e780178-d030-48c9-a7b1-6e930a95de68 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:54:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-4pzzs namespace: kserve-ci-e2e-test resourceVersion: "14518" uid: 9c79e632-ba98-4a09-ad37-0992ed0e90f1 kind: Event lastTimestamp: "2026-09-16T18:54:31Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-09-16T18:54:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:54:31Z" name: isvc-xgboost-graph-predictor-555fc69d56-4pzzs.18d5e1f59e89a1ef namespace: kserve-ci-e2e-test resourceVersion: "14642" uid: 2288678f-6606-4f45-af8d-5f76a4e8a772 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:55:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-4pzzs namespace: kserve-ci-e2e-test resourceVersion: "14518" uid: 9c79e632-ba98-4a09-ad37-0992ed0e90f1 kind: Event lastTimestamp: "2026-09-16T18:55:02Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 30.489s (30.489s including waiting). Image size: 1332365570 bytes.' metadata: creationTimestamp: "2026-09-16T18:55:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:55:02Z" name: isvc-xgboost-graph-predictor-555fc69d56-4pzzs.18d5e1fcb7d6d811 namespace: kserve-ci-e2e-test resourceVersion: "14944" uid: 3fe27097-a133-480c-949d-1fcf3b95867e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:55:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-4pzzs namespace: kserve-ci-e2e-test resourceVersion: "14518" uid: 9c79e632-ba98-4a09-ad37-0992ed0e90f1 kind: Event lastTimestamp: "2026-09-16T18:55:02Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T18:55:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:55:02Z" name: isvc-xgboost-graph-predictor-555fc69d56-4pzzs.18d5e1fcbd6385c6 namespace: kserve-ci-e2e-test resourceVersion: "14946" uid: ec9da1c0-3936-4d24-b01e-92de0f3a44c5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:55:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-4pzzs namespace: kserve-ci-e2e-test resourceVersion: "14518" uid: 9c79e632-ba98-4a09-ad37-0992ed0e90f1 kind: Event lastTimestamp: "2026-09-16T18:55:02Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T18:55:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:55:02Z" name: isvc-xgboost-graph-predictor-555fc69d56-4pzzs.18d5e1fcbe92fae8 namespace: kserve-ci-e2e-test resourceVersion: "14947" uid: 42e443f1-96c0-45ba-86a1-8104686f7fc0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:55:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-4pzzs namespace: kserve-ci-e2e-test resourceVersion: "14518" uid: 9c79e632-ba98-4a09-ad37-0992ed0e90f1 kind: Event lastTimestamp: "2026-09-16T18:55:02Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-16T18:55:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:55:02Z" name: isvc-xgboost-graph-predictor-555fc69d56-4pzzs.18d5e1fcbeaadb40 namespace: kserve-ci-e2e-test resourceVersion: "14949" uid: d53db641-2383-4216-beb2-c5bdf322cf38 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:55:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-4pzzs namespace: kserve-ci-e2e-test resourceVersion: "14518" uid: 9c79e632-ba98-4a09-ad37-0992ed0e90f1 kind: Event lastTimestamp: "2026-09-16T18:55:02Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T18:55:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:55:02Z" name: isvc-xgboost-graph-predictor-555fc69d56-4pzzs.18d5e1fcc3731cd5 namespace: kserve-ci-e2e-test resourceVersion: "14950" uid: b3c2b694-f2a4-41a9-a3b5-2d17bf984eb8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:55:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-4pzzs namespace: kserve-ci-e2e-test resourceVersion: "14518" uid: 9c79e632-ba98-4a09-ad37-0992ed0e90f1 kind: Event lastTimestamp: "2026-09-16T18:55:02Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T18:55:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:55:02Z" name: isvc-xgboost-graph-predictor-555fc69d56-4pzzs.18d5e1fcc526ab43 namespace: kserve-ci-e2e-test resourceVersion: "14951" uid: 6d2ae3bb-a156-4487-9d07-dd8d814c8553 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-09-16T18:55:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-4pzzs namespace: kserve-ci-e2e-test resourceVersion: "14518" uid: 9c79e632-ba98-4a09-ad37-0992ed0e90f1 kind: Event lastTimestamp: "2026-09-16T18:56:00Z" message: 'Readiness probe failed: dial tcp 10.134.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T18:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:00Z" name: isvc-xgboost-graph-predictor-555fc69d56-4pzzs.18d5e1fd1ddcd3fa namespace: kserve-ci-e2e-test resourceVersion: "15476" uid: f11892bf-a5db-4b88-80f0-bf490e8dc7ef reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-4pzzs namespace: kserve-ci-e2e-test resourceVersion: "14518" uid: 9c79e632-ba98-4a09-ad37-0992ed0e90f1 kind: Event lastTimestamp: "2026-09-16T18:56:45Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T18:56:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:45Z" name: isvc-xgboost-graph-predictor-555fc69d56-4pzzs.18d5e214c932f86b namespace: kserve-ci-e2e-test resourceVersion: "16119" uid: 9a556dbf-b0fb-4863-865f-83f5c9f7f240 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-4pzzs namespace: kserve-ci-e2e-test resourceVersion: "14518" uid: 9c79e632-ba98-4a09-ad37-0992ed0e90f1 kind: Event lastTimestamp: "2026-09-16T18:56:45Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T18:56:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:45Z" name: isvc-xgboost-graph-predictor-555fc69d56-4pzzs.18d5e214c93389f0 namespace: kserve-ci-e2e-test resourceVersion: "16121" uid: d2201903-af09-4721-bcf0-22a2cacc2299 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:54:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-555fc69d56 namespace: kserve-ci-e2e-test resourceVersion: "14504" uid: fb6d9844-e115-4c93-8354-5f5d2c6563ed kind: Event lastTimestamp: "2026-09-16T18:54:26Z" message: 'Created pod: isvc-xgboost-graph-predictor-555fc69d56-4pzzs' metadata: creationTimestamp: "2026-09-16T18:54:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:54:26Z" name: isvc-xgboost-graph-predictor-555fc69d56.18d5e1f43e09098e namespace: kserve-ci-e2e-test resourceVersion: "14517" uid: 60a6c6e6-7b30-4add-8dde-cd5aacd6f2aa reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:54:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14501" uid: ba408109-68f5-46db-98ce-62dc2d70c709 kind: Event lastTimestamp: "2026-09-16T18:54:25Z" message: Scaled up replica set isvc-xgboost-graph-predictor-555fc69d56 from 0 to 1 metadata: creationTimestamp: "2026-09-16T18:54:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:54:25Z" name: isvc-xgboost-graph-predictor.18d5e1f437951c05 namespace: kserve-ci-e2e-test resourceVersion: "14506" uid: 87995516-c522-40c3-85ca-df1a31e07e69 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-16T18:54:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14503" uid: 09dbb0aa-8641-4630-b68e-029f93ca9535 kind: Event lastTimestamp: "2026-09-16T18:55:41Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T18:54:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:55:41Z" name: isvc-xgboost-graph-predictor.18d5e1f7b61d3887 namespace: kserve-ci-e2e-test resourceVersion: "15278" uid: 00b4545f-469f-4434-87f4-9b8b0b25e637 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-16T18:54:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14503" uid: 09dbb0aa-8641-4630-b68e-029f93ca9535 kind: Event lastTimestamp: "2026-09-16T18:55:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T18:54:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:55:41Z" name: isvc-xgboost-graph-predictor.18d5e1f7b6861f5e namespace: kserve-ci-e2e-test resourceVersion: "15279" uid: d1dd3d0d-19e4-4afa-88ac-73efe02c351b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-16T18:55:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14739" uid: 09dbb0aa-8641-4630-b68e-029f93ca9535 kind: Event lastTimestamp: "2026-09-16T18:56:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T18:55:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:56:41Z" name: isvc-xgboost-graph-predictor.18d5e20930a7c427 namespace: kserve-ci-e2e-test resourceVersion: "16006" uid: 6f7bc942-6a1a-41ef-ae44-5cab1be5e37d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-16T18:55:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14739" uid: 09dbb0aa-8641-4630-b68e-029f93ca9535 kind: Event lastTimestamp: "2026-09-16T18:56:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T18:55:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:56:41Z" name: isvc-xgboost-graph-predictor.18d5e209310db87a namespace: kserve-ci-e2e-test resourceVersion: "16007" uid: ce91996c-c282-4465-b07c-baca2196f906 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-09-16T18:54:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14488" uid: 59ed0faa-521f-4d2f-b44e-69c7603fb2ec kind: Event lastTimestamp: "2026-09-16T18:56:10Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T18:54:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T18:56:10Z" name: isvc-xgboost-graph.18d5e1f437a5a0dd namespace: kserve-ci-e2e-test resourceVersion: "15722" uid: f350f90f-18a1-4772-8cba-42292457afd2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15720" uid: 59ed0faa-521f-4d2f-b44e-69c7603fb2ec kind: Event lastTimestamp: "2026-09-16T18:56:10Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-09-16T18:56:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T18:56:10Z" name: isvc-xgboost-graph.18d5e20c81ead798 namespace: kserve-ci-e2e-test resourceVersion: "15721" uid: b8e710ea-d40b-4b7a-9674-b86db9c2909d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T18:56:35.550262Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-648f748b4b-pjqp5 namespace: kserve-ci-e2e-test resourceVersion: "15934" uid: a3849260-af9d-4472-94b7-6b8834dd136b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-648f748b4b-pjqp5 to ip-10-0-129-152.ec2.internal metadata: creationTimestamp: "2026-09-16T18:56:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T18:56:35Z" name: model-chainer-648f748b4b-pjqp5.18d5e21263249ef1 namespace: kserve-ci-e2e-test resourceVersion: "15939" uid: c1803d8f-222e-442a-a9ea-5ec4973509bf reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-594cc48779-9d5dg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:36Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-648f748b4b-pjqp5 namespace: kserve-ci-e2e-test resourceVersion: "15942" uid: a3849260-af9d-4472-94b7-6b8834dd136b kind: Event lastTimestamp: "2026-09-16T18:56:36Z" message: Add eth0 [10.133.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T18:56:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T18:56:36Z" name: model-chainer-648f748b4b-pjqp5.18d5e21285a37291 namespace: kserve-ci-e2e-test resourceVersion: "15958" uid: 0d89e4a2-f978-43bd-a083-3621158f4857 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-648f748b4b-pjqp5 namespace: kserve-ci-e2e-test resourceVersion: "15936" uid: a3849260-af9d-4472-94b7-6b8834dd136b kind: Event lastTimestamp: "2026-09-16T18:56:36Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:10ee05d4acb8a93a3eeee1c000e44a3fdd4ba5c5852a56e3b2d09fe8b08cf42b" already present on machine metadata: creationTimestamp: "2026-09-16T18:56:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:36Z" name: model-chainer-648f748b4b-pjqp5.18d5e21286fb327a namespace: kserve-ci-e2e-test resourceVersion: "15961" uid: 8f954da1-1a7a-4478-9187-a06ac37073a5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-152.ec2.internal source: component: kubelet host: ip-10-0-129-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-648f748b4b-pjqp5 namespace: kserve-ci-e2e-test resourceVersion: "15936" uid: a3849260-af9d-4472-94b7-6b8834dd136b kind: Event lastTimestamp: "2026-09-16T18:56:36Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-09-16T18:56:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:36Z" name: model-chainer-648f748b4b-pjqp5.18d5e2128a00d583 namespace: kserve-ci-e2e-test resourceVersion: "15965" uid: 24e9fc25-f31a-49a9-8f00-12a18420f24a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-152.ec2.internal source: component: kubelet host: ip-10-0-129-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-648f748b4b-pjqp5 namespace: kserve-ci-e2e-test resourceVersion: "15936" uid: a3849260-af9d-4472-94b7-6b8834dd136b kind: Event lastTimestamp: "2026-09-16T18:56:36Z" message: Started container model-chainer metadata: creationTimestamp: "2026-09-16T18:56:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:36Z" name: model-chainer-648f748b4b-pjqp5.18d5e2128a722f07 namespace: kserve-ci-e2e-test resourceVersion: "15966" uid: cdebde8f-ac46-4a7f-a66a-cc18304d1ee2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-152.ec2.internal source: component: kubelet host: ip-10-0-129-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-648f748b4b-pjqp5 namespace: kserve-ci-e2e-test resourceVersion: "15936" uid: a3849260-af9d-4472-94b7-6b8834dd136b kind: Event lastTimestamp: "2026-09-16T18:56:45Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-09-16T18:56:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:45Z" name: model-chainer-648f748b4b-pjqp5.18d5e214bdb14da3 namespace: kserve-ci-e2e-test resourceVersion: "16063" uid: 0bacf368-96b6-45c0-b4fa-9e014312bd19 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-152.ec2.internal source: component: kubelet host: ip-10-0-129-152.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T18:56:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-648f748b4b-pjqp5 namespace: kserve-ci-e2e-test resourceVersion: "15936" uid: a3849260-af9d-4472-94b7-6b8834dd136b kind: Event lastTimestamp: "2026-09-16T18:57:12Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-16T18:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:57:12Z" name: model-chainer-648f748b4b-pjqp5.18d5e2152e21a0ec namespace: kserve-ci-e2e-test resourceVersion: "16480" uid: d98ec7ac-8450-4020-b574-5fda0f72a0bb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-152.ec2.internal source: component: kubelet host: ip-10-0-129-152.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-648f748b4b namespace: kserve-ci-e2e-test resourceVersion: "15931" uid: 1c4acc5b-a729-4811-b9e3-21d60ba1441b kind: Event lastTimestamp: "2026-09-16T18:56:35Z" message: 'Created pod: model-chainer-648f748b4b-pjqp5' metadata: creationTimestamp: "2026-09-16T18:56:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:56:35Z" name: model-chainer-648f748b4b.18d5e21262c34910 namespace: kserve-ci-e2e-test resourceVersion: "15937" uid: 693d69d7-69e1-492a-be2e-86697a380282 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15929" uid: 67e6b7fe-ae17-4846-b360-97588e2c2158 kind: Event lastTimestamp: "2026-09-16T18:56:35Z" message: Scaled up replica set model-chainer-648f748b4b from 0 to 1 metadata: creationTimestamp: "2026-09-16T18:56:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:56:35Z" name: model-chainer.18d5e21261d5cf07 namespace: kserve-ci-e2e-test resourceVersion: "15932" uid: b843d226-9bd7-40d7-aa3a-066098eeb8bb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:35Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15946" uid: 8c3747bf-2b0d-43d1-a864-ad79da44e5aa kind: Event lastTimestamp: "2026-09-16T18:56:35Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-09-16T18:56:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T18:56:35Z" name: model-chainer.18d5e212673aa91f namespace: kserve-ci-e2e-test resourceVersion: "15948" uid: 0fc61986-3dcf-42e0-b9c2-d79913c2233f reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T18:57:45.888069Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-2796f-5db7b6657f-b9tqk namespace: kserve-ci-e2e-test resourceVersion: "16769" uid: a7e662bf-694b-4262-9251-02a95f9f067d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-2796f-5db7b6657f-b9tqk to ip-10-0-129-152.ec2.internal metadata: creationTimestamp: "2026-09-16T18:57:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T18:57:45Z" name: sequence-graph-2796f-5db7b6657f-b9tqk.18d5e222c39a5b34 namespace: kserve-ci-e2e-test resourceVersion: "16774" uid: b77b1e61-dc0d-4666-9f40-b16436c7c385 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-594cc48779-9d5dg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:46Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-2796f-5db7b6657f-b9tqk namespace: kserve-ci-e2e-test resourceVersion: "16778" uid: a7e662bf-694b-4262-9251-02a95f9f067d kind: Event lastTimestamp: "2026-09-16T18:57:46Z" message: Add eth0 [10.133.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T18:57:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T18:57:46Z" name: sequence-graph-2796f-5db7b6657f-b9tqk.18d5e222e5691a34 namespace: kserve-ci-e2e-test resourceVersion: "16793" uid: 11a3f888-5d92-4fa6-adb2-8ab7ceb50dba reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-2796f} kind: Pod name: sequence-graph-2796f-5db7b6657f-b9tqk namespace: kserve-ci-e2e-test resourceVersion: "16771" uid: a7e662bf-694b-4262-9251-02a95f9f067d kind: Event lastTimestamp: "2026-09-16T18:57:46Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:10ee05d4acb8a93a3eeee1c000e44a3fdd4ba5c5852a56e3b2d09fe8b08cf42b" already present on machine metadata: creationTimestamp: "2026-09-16T18:57:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:57:46Z" name: sequence-graph-2796f-5db7b6657f-b9tqk.18d5e222e6c75b64 namespace: kserve-ci-e2e-test resourceVersion: "16795" uid: 9abf81ae-63b3-4960-a072-496325f0f128 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-152.ec2.internal source: component: kubelet host: ip-10-0-129-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-2796f} kind: Pod name: sequence-graph-2796f-5db7b6657f-b9tqk namespace: kserve-ci-e2e-test resourceVersion: "16771" uid: a7e662bf-694b-4262-9251-02a95f9f067d kind: Event lastTimestamp: "2026-09-16T18:57:46Z" message: 'Created container: sequence-graph-2796f' metadata: creationTimestamp: "2026-09-16T18:57:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:57:46Z" name: sequence-graph-2796f-5db7b6657f-b9tqk.18d5e222ea0bd1ee namespace: kserve-ci-e2e-test resourceVersion: "16796" uid: abc1acb7-d9ae-4961-b3f9-0d9bddc4e27e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-152.ec2.internal source: component: kubelet host: ip-10-0-129-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-2796f} kind: Pod name: sequence-graph-2796f-5db7b6657f-b9tqk namespace: kserve-ci-e2e-test resourceVersion: "16771" uid: a7e662bf-694b-4262-9251-02a95f9f067d kind: Event lastTimestamp: "2026-09-16T18:57:46Z" message: Started container sequence-graph-2796f metadata: creationTimestamp: "2026-09-16T18:57:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:57:46Z" name: sequence-graph-2796f-5db7b6657f-b9tqk.18d5e222ea7b65e7 namespace: kserve-ci-e2e-test resourceVersion: "16797" uid: daf85b0a-f4fc-47d9-850f-22a68bf354bc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-152.ec2.internal source: component: kubelet host: ip-10-0-129-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-2796f} kind: Pod name: sequence-graph-2796f-5db7b6657f-b9tqk namespace: kserve-ci-e2e-test resourceVersion: "16771" uid: a7e662bf-694b-4262-9251-02a95f9f067d kind: Event lastTimestamp: "2026-09-16T19:06:00Z" message: Stopping container sequence-graph-2796f metadata: creationTimestamp: "2026-09-16T19:06:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:06:00Z" name: sequence-graph-2796f-5db7b6657f-b9tqk.18d5e295ec286abd namespace: kserve-ci-e2e-test resourceVersion: "20351" uid: 9e6e2dff-01e4-43ae-8951-00f8799b614f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-152.ec2.internal source: component: kubelet host: ip-10-0-129-152.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T19:06:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-2796f} kind: Pod name: sequence-graph-2796f-5db7b6657f-b9tqk namespace: kserve-ci-e2e-test resourceVersion: "16771" uid: a7e662bf-694b-4262-9251-02a95f9f067d kind: Event lastTimestamp: "2026-09-16T19:06:27Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-16T19:06:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:06:27Z" name: sequence-graph-2796f-5db7b6657f-b9tqk.18d5e29679e1bd6b namespace: kserve-ci-e2e-test resourceVersion: "20753" uid: faa2b413-ac19-41ab-8915-cf106ae86657 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-152.ec2.internal source: component: kubelet host: ip-10-0-129-152.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-2796f-5db7b6657f namespace: kserve-ci-e2e-test resourceVersion: "16766" uid: 799e95c3-3d60-47e4-bf50-13ec4e79157d kind: Event lastTimestamp: "2026-09-16T18:57:45Z" message: 'Created pod: sequence-graph-2796f-5db7b6657f-b9tqk' metadata: creationTimestamp: "2026-09-16T18:57:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:57:45Z" name: sequence-graph-2796f-5db7b6657f.18d5e222c2f2702a namespace: kserve-ci-e2e-test resourceVersion: "16770" uid: a3fcce34-cd86-4476-9fc4-e2076291a67a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-2796f namespace: kserve-ci-e2e-test resourceVersion: "16764" uid: b5a1cddb-485d-49dd-9a68-fef6693294b6 kind: Event lastTimestamp: "2026-09-16T18:57:45Z" message: Scaled up replica set sequence-graph-2796f-5db7b6657f from 0 to 1 metadata: creationTimestamp: "2026-09-16T18:57:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:57:45Z" name: sequence-graph-2796f.18d5e222c201d819 namespace: kserve-ci-e2e-test resourceVersion: "16767" uid: 85763699-64da-49d5-ae87-25b0f759d709 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:45Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-2796f namespace: kserve-ci-e2e-test resourceVersion: "16780" uid: 0c028417-65e3-4418-a2d5-a2c62a6a7496 kind: Event lastTimestamp: "2026-09-16T18:57:45Z" message: InferenceGraph [sequence-graph-2796f] is Ready metadata: creationTimestamp: "2026-09-16T18:57:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T18:57:45Z" name: sequence-graph-2796f.18d5e222c945eec7 namespace: kserve-ci-e2e-test resourceVersion: "16782" uid: 770d5f8d-c4e9-4ca4-aee6-c3327748b9c7 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T18:58:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-2796f namespace: kserve-ci-e2e-test resourceVersion: "16765" uid: 52f91842-fca5-4706-b554-31dfd03cd303 kind: Event lastTimestamp: "2026-09-16T18:58:15Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T18:58:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:58:15Z" name: sequence-graph-2796f.18d5e226408d158d namespace: kserve-ci-e2e-test resourceVersion: "17015" uid: e0b5433d-bcb5-4c54-8ad7-5e6a1e3307c9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T18:58:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-2796f namespace: kserve-ci-e2e-test resourceVersion: "16765" uid: 52f91842-fca5-4706-b554-31dfd03cd303 kind: Event lastTimestamp: "2026-09-16T18:58:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T18:58:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:58:15Z" name: sequence-graph-2796f.18d5e22640f902d3 namespace: kserve-ci-e2e-test resourceVersion: "17016" uid: 443e5553-3ed7-4930-9fa9-0db19b49a17d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:08:11.187040Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-e99c4-95c6d94f8-tgdbt namespace: kserve-ci-e2e-test resourceVersion: "22019" uid: ebe0ed08-8180-48ca-a889-ac126d64bf25 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-e99c4-95c6d94f8-tgdbt to ip-10-0-129-152.ec2.internal metadata: creationTimestamp: "2026-09-16T19:08:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:08:11Z" name: sequence-graph-e99c4-95c6d94f8-tgdbt.18d5e2b45a5367d5 namespace: kserve-ci-e2e-test resourceVersion: "22027" uid: a6b2b116-dba3-4244-b791-17c263055110 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-594cc48779-9d5dg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:11Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-e99c4-95c6d94f8-tgdbt namespace: kserve-ci-e2e-test resourceVersion: "22032" uid: ebe0ed08-8180-48ca-a889-ac126d64bf25 kind: Event lastTimestamp: "2026-09-16T19:08:11Z" message: Add eth0 [10.133.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:08:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:08:11Z" name: sequence-graph-e99c4-95c6d94f8-tgdbt.18d5e2b47bfdb82e namespace: kserve-ci-e2e-test resourceVersion: "22043" uid: c364eb88-ff91-474f-960c-eb71377a1e85 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-e99c4} kind: Pod name: sequence-graph-e99c4-95c6d94f8-tgdbt namespace: kserve-ci-e2e-test resourceVersion: "22024" uid: ebe0ed08-8180-48ca-a889-ac126d64bf25 kind: Event lastTimestamp: "2026-09-16T19:08:11Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:10ee05d4acb8a93a3eeee1c000e44a3fdd4ba5c5852a56e3b2d09fe8b08cf42b" already present on machine metadata: creationTimestamp: "2026-09-16T19:08:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:11Z" name: sequence-graph-e99c4-95c6d94f8-tgdbt.18d5e2b47d592abb namespace: kserve-ci-e2e-test resourceVersion: "22045" uid: 282c7bc3-c6a5-48a0-9a42-745c6b6a81fb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-152.ec2.internal source: component: kubelet host: ip-10-0-129-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-e99c4} kind: Pod name: sequence-graph-e99c4-95c6d94f8-tgdbt namespace: kserve-ci-e2e-test resourceVersion: "22024" uid: ebe0ed08-8180-48ca-a889-ac126d64bf25 kind: Event lastTimestamp: "2026-09-16T19:08:11Z" message: 'Created container: sequence-graph-e99c4' metadata: creationTimestamp: "2026-09-16T19:08:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:11Z" name: sequence-graph-e99c4-95c6d94f8-tgdbt.18d5e2b480a2dca8 namespace: kserve-ci-e2e-test resourceVersion: "22047" uid: 34d8c04b-f078-441e-bea9-ca94407fc19d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-152.ec2.internal source: component: kubelet host: ip-10-0-129-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-e99c4} kind: Pod name: sequence-graph-e99c4-95c6d94f8-tgdbt namespace: kserve-ci-e2e-test resourceVersion: "22024" uid: ebe0ed08-8180-48ca-a889-ac126d64bf25 kind: Event lastTimestamp: "2026-09-16T19:08:11Z" message: Started container sequence-graph-e99c4 metadata: creationTimestamp: "2026-09-16T19:08:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:11Z" name: sequence-graph-e99c4-95c6d94f8-tgdbt.18d5e2b481113d3d namespace: kserve-ci-e2e-test resourceVersion: "22048" uid: b6e39f85-98e0-4c5b-b9c4-8f7dd5c67c93 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-152.ec2.internal source: component: kubelet host: ip-10-0-129-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-e99c4} kind: Pod name: sequence-graph-e99c4-95c6d94f8-tgdbt namespace: kserve-ci-e2e-test resourceVersion: "22024" uid: ebe0ed08-8180-48ca-a889-ac126d64bf25 kind: Event lastTimestamp: "2026-09-16T19:16:25Z" message: Stopping container sequence-graph-e99c4 metadata: creationTimestamp: "2026-09-16T19:16:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:16:25Z" name: sequence-graph-e99c4-95c6d94f8-tgdbt.18d5e3277a208eec namespace: kserve-ci-e2e-test resourceVersion: "25631" uid: f71c2ac2-5c0d-4bd9-98f6-7235bf12c799 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-152.ec2.internal source: component: kubelet host: ip-10-0-129-152.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T19:16:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-e99c4} kind: Pod name: sequence-graph-e99c4-95c6d94f8-tgdbt namespace: kserve-ci-e2e-test resourceVersion: "22024" uid: ebe0ed08-8180-48ca-a889-ac126d64bf25 kind: Event lastTimestamp: "2026-09-16T19:16:53Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-16T19:16:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:16:53Z" name: sequence-graph-e99c4-95c6d94f8-tgdbt.18d5e3283674dcb0 namespace: kserve-ci-e2e-test resourceVersion: "26335" uid: 4e5ddb36-2149-4d35-b0c1-09b6eb6ebeeb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-152.ec2.internal source: component: kubelet host: ip-10-0-129-152.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-e99c4-95c6d94f8 namespace: kserve-ci-e2e-test resourceVersion: "22015" uid: d035ebf5-2572-4d84-a440-772a915ea4ea kind: Event lastTimestamp: "2026-09-16T19:08:11Z" message: 'Created pod: sequence-graph-e99c4-95c6d94f8-tgdbt' metadata: creationTimestamp: "2026-09-16T19:08:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:08:11Z" name: sequence-graph-e99c4-95c6d94f8.18d5e2b458947a5c namespace: kserve-ci-e2e-test resourceVersion: "22022" uid: 4a776407-7f14-4b75-9d2a-ca074cd7f009 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-e99c4 namespace: kserve-ci-e2e-test resourceVersion: "22013" uid: 9822c901-331c-41a3-896b-747189175a20 kind: Event lastTimestamp: "2026-09-16T19:08:11Z" message: Scaled up replica set sequence-graph-e99c4-95c6d94f8 from 0 to 1 metadata: creationTimestamp: "2026-09-16T19:08:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:08:11Z" name: sequence-graph-e99c4.18d5e2b4559eca13 namespace: kserve-ci-e2e-test resourceVersion: "22016" uid: ce975223-c9dc-4a70-94b0-fd9f21802e83 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:11Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-e99c4 namespace: kserve-ci-e2e-test resourceVersion: "22036" uid: 944e02f9-7229-426f-ac3a-c98d02d06a56 kind: Event lastTimestamp: "2026-09-16T19:08:11Z" message: InferenceGraph [sequence-graph-e99c4] is Ready metadata: creationTimestamp: "2026-09-16T19:08:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:08:11Z" name: sequence-graph-e99c4.18d5e2b45f710585 namespace: kserve-ci-e2e-test resourceVersion: "22038" uid: 49c14244-1a76-42cd-a0e4-186a309fd2e6 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-e99c4 namespace: kserve-ci-e2e-test resourceVersion: "22014" uid: 51b73504-732c-46d3-8d29-4a396f37a977 kind: Event lastTimestamp: "2026-09-16T19:08:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:08:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:08:26Z" name: sequence-graph-e99c4.18d5e2b7d3c2d93c namespace: kserve-ci-e2e-test resourceVersion: "22153" uid: b7839103-0476-4959-bb6c-db38adf1542c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-e99c4 namespace: kserve-ci-e2e-test resourceVersion: "22014" uid: 51b73504-732c-46d3-8d29-4a396f37a977 kind: Event lastTimestamp: "2026-09-16T19:08:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:08:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:08:26Z" name: sequence-graph-e99c4.18d5e2b7d42d5731 namespace: kserve-ci-e2e-test resourceVersion: "22155" uid: a2327ce3-accd-4f8f-a929-97b67d393f11 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-e99c4 namespace: kserve-ci-e2e-test resourceVersion: "22154" uid: 51b73504-732c-46d3-8d29-4a396f37a977 kind: Event lastTimestamp: "2026-09-16T19:08:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:08:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:08:41Z" name: sequence-graph-e99c4.18d5e2bb52e645ca namespace: kserve-ci-e2e-test resourceVersion: "22265" uid: 72c53575-7ca4-4e38-a6b8-54f83e535c1b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-e99c4 namespace: kserve-ci-e2e-test resourceVersion: "22154" uid: 51b73504-732c-46d3-8d29-4a396f37a977 kind: Event lastTimestamp: "2026-09-16T19:08:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:08:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:08:41Z" name: sequence-graph-e99c4.18d5e2bb533b6f75 namespace: kserve-ci-e2e-test resourceVersion: "22267" uid: e702740a-ddb4-4863-8362-bffeb54265c9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:07:00.728152Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-f64b9-59cc8f9984-ssqv5 namespace: kserve-ci-e2e-test resourceVersion: "21204" uid: 14a6323e-d9a4-42ed-b646-93dc7db72be2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-f64b9-59cc8f9984-ssqv5 to ip-10-0-129-152.ec2.internal metadata: creationTimestamp: "2026-09-16T19:07:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:07:00Z" name: sequence-graph-f64b9-59cc8f9984-ssqv5.18d5e2a3f2a611dd namespace: kserve-ci-e2e-test resourceVersion: "21209" uid: 9c68f793-a521-462c-ad92-4cc6b085e546 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-594cc48779-9d5dg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:00Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-f64b9-59cc8f9984-ssqv5 namespace: kserve-ci-e2e-test resourceVersion: "21206" uid: 14a6323e-d9a4-42ed-b646-93dc7db72be2 kind: Event lastTimestamp: "2026-09-16T19:07:00Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-f64b9-serving-cert" not found' metadata: creationTimestamp: "2026-09-16T19:07:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:00Z" name: sequence-graph-f64b9-59cc8f9984-ssqv5.18d5e2a3fc5aecbd namespace: kserve-ci-e2e-test resourceVersion: "21222" uid: b9841a37-d5c6-4292-984b-f5dae6606a84 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-152.ec2.internal source: component: kubelet host: ip-10-0-129-152.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:01Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-f64b9-59cc8f9984-ssqv5 namespace: kserve-ci-e2e-test resourceVersion: "21212" uid: 14a6323e-d9a4-42ed-b646-93dc7db72be2 kind: Event lastTimestamp: "2026-09-16T19:07:01Z" message: Add eth0 [10.133.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:07:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:07:01Z" name: sequence-graph-f64b9-59cc8f9984-ssqv5.18d5e2a4386e121e namespace: kserve-ci-e2e-test resourceVersion: "21246" uid: e6291ade-3df8-4f07-b7ec-2b04f774dcf7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f64b9} kind: Pod name: sequence-graph-f64b9-59cc8f9984-ssqv5 namespace: kserve-ci-e2e-test resourceVersion: "21206" uid: 14a6323e-d9a4-42ed-b646-93dc7db72be2 kind: Event lastTimestamp: "2026-09-16T19:07:01Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:10ee05d4acb8a93a3eeee1c000e44a3fdd4ba5c5852a56e3b2d09fe8b08cf42b" already present on machine metadata: creationTimestamp: "2026-09-16T19:07:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:01Z" name: sequence-graph-f64b9-59cc8f9984-ssqv5.18d5e2a439f394d8 namespace: kserve-ci-e2e-test resourceVersion: "21248" uid: 4cc1092f-bac1-4881-ad8c-921d1ae91f27 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-152.ec2.internal source: component: kubelet host: ip-10-0-129-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f64b9} kind: Pod name: sequence-graph-f64b9-59cc8f9984-ssqv5 namespace: kserve-ci-e2e-test resourceVersion: "21206" uid: 14a6323e-d9a4-42ed-b646-93dc7db72be2 kind: Event lastTimestamp: "2026-09-16T19:07:01Z" message: 'Created container: sequence-graph-f64b9' metadata: creationTimestamp: "2026-09-16T19:07:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:01Z" name: sequence-graph-f64b9-59cc8f9984-ssqv5.18d5e2a43cd53ca3 namespace: kserve-ci-e2e-test resourceVersion: "21250" uid: 1eb97649-b180-4f16-94dd-af7f206d05b7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-152.ec2.internal source: component: kubelet host: ip-10-0-129-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f64b9} kind: Pod name: sequence-graph-f64b9-59cc8f9984-ssqv5 namespace: kserve-ci-e2e-test resourceVersion: "21206" uid: 14a6323e-d9a4-42ed-b646-93dc7db72be2 kind: Event lastTimestamp: "2026-09-16T19:07:01Z" message: Started container sequence-graph-f64b9 metadata: creationTimestamp: "2026-09-16T19:07:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:01Z" name: sequence-graph-f64b9-59cc8f9984-ssqv5.18d5e2a43d423bc7 namespace: kserve-ci-e2e-test resourceVersion: "21251" uid: aa691111-7dfb-48e1-8e67-8bf52c152d41 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-152.ec2.internal source: component: kubelet host: ip-10-0-129-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f64b9} kind: Pod name: sequence-graph-f64b9-59cc8f9984-ssqv5 namespace: kserve-ci-e2e-test resourceVersion: "21206" uid: 14a6323e-d9a4-42ed-b646-93dc7db72be2 kind: Event lastTimestamp: "2026-09-16T19:07:10Z" message: Stopping container sequence-graph-f64b9 metadata: creationTimestamp: "2026-09-16T19:07:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:10Z" name: sequence-graph-f64b9-59cc8f9984-ssqv5.18d5e2a64b70fc34 namespace: kserve-ci-e2e-test resourceVersion: "21331" uid: 61e08f96-6d2c-4db2-8ecc-a2c4439f2529 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-152.ec2.internal source: component: kubelet host: ip-10-0-129-152.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T19:07:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f64b9} kind: Pod name: sequence-graph-f64b9-59cc8f9984-ssqv5 namespace: kserve-ci-e2e-test resourceVersion: "21206" uid: 14a6323e-d9a4-42ed-b646-93dc7db72be2 kind: Event lastTimestamp: "2026-09-16T19:07:38Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-16T19:07:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:38Z" name: sequence-graph-f64b9-59cc8f9984-ssqv5.18d5e2a6e933002d namespace: kserve-ci-e2e-test resourceVersion: "21733" uid: 3b4d41c9-f238-4d62-be0b-58a46267ce6e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-152.ec2.internal source: component: kubelet host: ip-10-0-129-152.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-f64b9-59cc8f9984 namespace: kserve-ci-e2e-test resourceVersion: "21201" uid: dfd8e0e4-dbce-4efe-869c-ee8b802bd867 kind: Event lastTimestamp: "2026-09-16T19:07:00Z" message: 'Created pod: sequence-graph-f64b9-59cc8f9984-ssqv5' metadata: creationTimestamp: "2026-09-16T19:07:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:07:00Z" name: sequence-graph-f64b9-59cc8f9984.18d5e2a3f1db1e2c namespace: kserve-ci-e2e-test resourceVersion: "21207" uid: e682277c-80e8-44e7-9a5e-ea97bbc0fb2f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-f64b9 namespace: kserve-ci-e2e-test resourceVersion: "21199" uid: 85ea1e46-3078-4627-a067-01aa5d5fa4f4 kind: Event lastTimestamp: "2026-09-16T19:07:00Z" message: Scaled up replica set sequence-graph-f64b9-59cc8f9984 from 0 to 1 metadata: creationTimestamp: "2026-09-16T19:07:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:07:00Z" name: sequence-graph-f64b9.18d5e2a3f0d16c18 namespace: kserve-ci-e2e-test resourceVersion: "21202" uid: 53b59e0e-9cc8-4949-ab93-fe79ad9f97de reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:00Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-f64b9 namespace: kserve-ci-e2e-test resourceVersion: "21216" uid: e3641460-9ab9-457e-8b73-215edc7e9f0f kind: Event lastTimestamp: "2026-09-16T19:07:00Z" message: InferenceGraph [sequence-graph-f64b9] is Ready metadata: creationTimestamp: "2026-09-16T19:07:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:07:00Z" name: sequence-graph-f64b9.18d5e2a3f7938e7f namespace: kserve-ci-e2e-test resourceVersion: "21219" uid: c5ac2717-521e-4e6f-a928-ef94d8232f92 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:00Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-f64b9 namespace: kserve-ci-e2e-test resourceVersion: "21194" uid: e3641460-9ab9-457e-8b73-215edc7e9f0f kind: Event lastTimestamp: "2026-09-16T19:07:00Z" message: 'Failed to update status for InferenceGraph "sequence-graph-f64b9": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-f64b9": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-16T19:07:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:07:00Z" name: sequence-graph-f64b9.18d5e2a3fae0e91a namespace: kserve-ci-e2e-test resourceVersion: "21220" uid: 18f73c0c-08ad-4f8e-84d9-f976402ff636 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:00Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-f64b9 namespace: kserve-ci-e2e-test resourceVersion: "21194" uid: e3641460-9ab9-457e-8b73-215edc7e9f0f kind: Event lastTimestamp: "2026-09-16T19:07:00Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-f64b9": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-16T19:07:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:07:00Z" name: sequence-graph-f64b9.18d5e2a3fae1448d namespace: kserve-ci-e2e-test resourceVersion: "21221" uid: a6673350-07aa-4964-913d-cbe45a0ffd60 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:17:56.390830Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-4a90a-5d7c796cdb-xsjm4 namespace: kserve-ci-e2e-test resourceVersion: "27066" uid: 3c7f2418-d595-47ab-8e5d-1373653804fd kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-4a90a-5d7c796cdb-xsjm4 to ip-10-0-129-152.ec2.internal metadata: creationTimestamp: "2026-09-16T19:17:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:17:56Z" name: splitter-graph-4a90a-5d7c796cdb-xsjm4.18d5e33c9b30a10a namespace: kserve-ci-e2e-test resourceVersion: "27072" uid: 64461b54-2009-4fb7-95d7-4dd7e731ef1b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-594cc48779-9d5dg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:17:56Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-4a90a-5d7c796cdb-xsjm4 namespace: kserve-ci-e2e-test resourceVersion: "27069" uid: 3c7f2418-d595-47ab-8e5d-1373653804fd kind: Event lastTimestamp: "2026-09-16T19:17:56Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-4a90a-serving-cert" not found' metadata: creationTimestamp: "2026-09-16T19:17:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:17:56Z" name: splitter-graph-4a90a-5d7c796cdb-xsjm4.18d5e33ca58fb92a namespace: kserve-ci-e2e-test resourceVersion: "27086" uid: 78d31f1d-c1c9-4a06-981f-24458ca3d054 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-152.ec2.internal source: component: kubelet host: ip-10-0-129-152.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:17:57Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-4a90a-5d7c796cdb-xsjm4 namespace: kserve-ci-e2e-test resourceVersion: "27076" uid: 3c7f2418-d595-47ab-8e5d-1373653804fd kind: Event lastTimestamp: "2026-09-16T19:17:57Z" message: Add eth0 [10.133.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:17:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:17:57Z" name: splitter-graph-4a90a-5d7c796cdb-xsjm4.18d5e33ce1056baa namespace: kserve-ci-e2e-test resourceVersion: "27095" uid: f8a52ccd-b85e-4bf3-be2b-7c1041150ca6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:17:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-4a90a} kind: Pod name: splitter-graph-4a90a-5d7c796cdb-xsjm4 namespace: kserve-ci-e2e-test resourceVersion: "27069" uid: 3c7f2418-d595-47ab-8e5d-1373653804fd kind: Event lastTimestamp: "2026-09-16T19:17:57Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:10ee05d4acb8a93a3eeee1c000e44a3fdd4ba5c5852a56e3b2d09fe8b08cf42b" already present on machine metadata: creationTimestamp: "2026-09-16T19:17:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:17:57Z" name: splitter-graph-4a90a-5d7c796cdb-xsjm4.18d5e33ce23decb0 namespace: kserve-ci-e2e-test resourceVersion: "27097" uid: 8b34ba9a-f4e1-48fe-b60c-8517adecfe1f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-152.ec2.internal source: component: kubelet host: ip-10-0-129-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:17:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-4a90a} kind: Pod name: splitter-graph-4a90a-5d7c796cdb-xsjm4 namespace: kserve-ci-e2e-test resourceVersion: "27069" uid: 3c7f2418-d595-47ab-8e5d-1373653804fd kind: Event lastTimestamp: "2026-09-16T19:17:57Z" message: 'Created container: splitter-graph-4a90a' metadata: creationTimestamp: "2026-09-16T19:17:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:17:57Z" name: splitter-graph-4a90a-5d7c796cdb-xsjm4.18d5e33ce52999bf namespace: kserve-ci-e2e-test resourceVersion: "27099" uid: fb1f6c97-e8d0-4bbe-8be0-1e8aa8941c22 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-152.ec2.internal source: component: kubelet host: ip-10-0-129-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:17:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-4a90a} kind: Pod name: splitter-graph-4a90a-5d7c796cdb-xsjm4 namespace: kserve-ci-e2e-test resourceVersion: "27069" uid: 3c7f2418-d595-47ab-8e5d-1373653804fd kind: Event lastTimestamp: "2026-09-16T19:17:57Z" message: Started container splitter-graph-4a90a metadata: creationTimestamp: "2026-09-16T19:17:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:17:57Z" name: splitter-graph-4a90a-5d7c796cdb-xsjm4.18d5e33ce5982a9d namespace: kserve-ci-e2e-test resourceVersion: "27100" uid: 2a675050-2bd8-461e-b7d7-d44d3d95bdbd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-152.ec2.internal source: component: kubelet host: ip-10-0-129-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:26:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-4a90a} kind: Pod name: splitter-graph-4a90a-5d7c796cdb-xsjm4 namespace: kserve-ci-e2e-test resourceVersion: "27069" uid: 3c7f2418-d595-47ab-8e5d-1373653804fd kind: Event lastTimestamp: "2026-09-16T19:26:10Z" message: Stopping container splitter-graph-4a90a metadata: creationTimestamp: "2026-09-16T19:26:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:26:10Z" name: splitter-graph-4a90a-5d7c796cdb-xsjm4.18d5e3afc1f8dde2 namespace: kserve-ci-e2e-test resourceVersion: "30519" uid: 90616582-5bb9-480b-881c-25de4b6e5112 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-152.ec2.internal source: component: kubelet host: ip-10-0-129-152.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T19:26:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-4a90a} kind: Pod name: splitter-graph-4a90a-5d7c796cdb-xsjm4 namespace: kserve-ci-e2e-test resourceVersion: "27069" uid: 3c7f2418-d595-47ab-8e5d-1373653804fd kind: Event lastTimestamp: "2026-09-16T19:26:39Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-16T19:26:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:26:39Z" name: splitter-graph-4a90a-5d7c796cdb-xsjm4.18d5e3b094fb6e6a namespace: kserve-ci-e2e-test resourceVersion: "30762" uid: bda96584-51bc-4adf-9efe-7daeb14e2443 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-152.ec2.internal source: component: kubelet host: ip-10-0-129-152.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:17:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-4a90a-5d7c796cdb namespace: kserve-ci-e2e-test resourceVersion: "27063" uid: e506c46b-9b92-4e75-acd1-98a1cbb28b94 kind: Event lastTimestamp: "2026-09-16T19:17:56Z" message: 'Created pod: splitter-graph-4a90a-5d7c796cdb-xsjm4' metadata: creationTimestamp: "2026-09-16T19:17:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:17:56Z" name: splitter-graph-4a90a-5d7c796cdb.18d5e33c9ab089a5 namespace: kserve-ci-e2e-test resourceVersion: "27068" uid: 622fb9c8-3774-43fb-a6bd-60d57e0ccc8c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:17:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-4a90a namespace: kserve-ci-e2e-test resourceVersion: "27061" uid: 5c8b1416-b61e-4314-887f-a15e4833612f kind: Event lastTimestamp: "2026-09-16T19:17:56Z" message: Scaled up replica set splitter-graph-4a90a-5d7c796cdb from 0 to 1 metadata: creationTimestamp: "2026-09-16T19:17:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:17:56Z" name: splitter-graph-4a90a.18d5e33c99563c0f namespace: kserve-ci-e2e-test resourceVersion: "27065" uid: e0a34ed4-2016-4c1f-9ee3-dfcd4b29836d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:17:56Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-4a90a namespace: kserve-ci-e2e-test resourceVersion: "27080" uid: d2b5cc48-1fe5-4507-960f-893f1681ab7b kind: Event lastTimestamp: "2026-09-16T19:17:56Z" message: InferenceGraph [splitter-graph-4a90a] is Ready metadata: creationTimestamp: "2026-09-16T19:17:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:17:56Z" name: splitter-graph-4a90a.18d5e33c9f869dc0 namespace: kserve-ci-e2e-test resourceVersion: "27082" uid: 874ba7a8-9b67-460c-9625-d7e27eca977b reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:18:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-4a90a namespace: kserve-ci-e2e-test resourceVersion: "27062" uid: dd6330d9-c38c-4f58-8b47-adab2b2cd75d kind: Event lastTimestamp: "2026-09-16T19:18:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:18:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:18:11Z" name: splitter-graph-4a90a.18d5e34017e2d491 namespace: kserve-ci-e2e-test resourceVersion: "27209" uid: b9757e93-84b4-4a6b-916b-30fd499c79a5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:18:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-4a90a namespace: kserve-ci-e2e-test resourceVersion: "27062" uid: dd6330d9-c38c-4f58-8b47-adab2b2cd75d kind: Event lastTimestamp: "2026-09-16T19:18:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:18:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:18:11Z" name: splitter-graph-4a90a.18d5e34018771bfa namespace: kserve-ci-e2e-test resourceVersion: "27211" uid: 0d84fe58-43b4-4270-ba30-9e053f828f89 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:18:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-4a90a namespace: kserve-ci-e2e-test resourceVersion: "27210" uid: dd6330d9-c38c-4f58-8b47-adab2b2cd75d kind: Event lastTimestamp: "2026-09-16T19:18:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:18:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:18:26Z" name: splitter-graph-4a90a.18d5e3439733412e namespace: kserve-ci-e2e-test resourceVersion: "27319" uid: 5a2a969e-a241-41f5-913f-c0a1b674331f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:18:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-4a90a namespace: kserve-ci-e2e-test resourceVersion: "27210" uid: dd6330d9-c38c-4f58-8b47-adab2b2cd75d kind: Event lastTimestamp: "2026-09-16T19:18:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:18:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:18:26Z" name: splitter-graph-4a90a.18d5e34397a37e77 namespace: kserve-ci-e2e-test resourceVersion: "27321" uid: 6c9c16b6-04e5-4d65-ba2f-db20894c873c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:16:46.069913Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-da3eb-5b6db7747c-28d44 namespace: kserve-ci-e2e-test resourceVersion: "25957" uid: 2ec43c15-7101-4ecd-b712-19ef88f41b1b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-da3eb-5b6db7747c-28d44 to ip-10-0-129-152.ec2.internal metadata: creationTimestamp: "2026-09-16T19:16:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:16:46Z" name: splitter-graph-da3eb-5b6db7747c-28d44.18d5e32c3bbc8bb1 namespace: kserve-ci-e2e-test resourceVersion: "25964" uid: 6a933736-5216-47e6-9d49-8d6ba545a76a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-594cc48779-9d5dg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:46Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-da3eb-5b6db7747c-28d44 namespace: kserve-ci-e2e-test resourceVersion: "25967" uid: 2ec43c15-7101-4ecd-b712-19ef88f41b1b kind: Event lastTimestamp: "2026-09-16T19:16:46Z" message: Add eth0 [10.133.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:16:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:16:46Z" name: splitter-graph-da3eb-5b6db7747c-28d44.18d5e32c5d9e013a namespace: kserve-ci-e2e-test resourceVersion: "25984" uid: cf0d5599-d9b5-47e6-b52b-757ca990e33c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-da3eb} kind: Pod name: splitter-graph-da3eb-5b6db7747c-28d44 namespace: kserve-ci-e2e-test resourceVersion: "25960" uid: 2ec43c15-7101-4ecd-b712-19ef88f41b1b kind: Event lastTimestamp: "2026-09-16T19:16:46Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:10ee05d4acb8a93a3eeee1c000e44a3fdd4ba5c5852a56e3b2d09fe8b08cf42b" already present on machine metadata: creationTimestamp: "2026-09-16T19:16:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:16:46Z" name: splitter-graph-da3eb-5b6db7747c-28d44.18d5e32c5f0b6f00 namespace: kserve-ci-e2e-test resourceVersion: "25987" uid: cd1e8614-2493-4c1f-98ea-460486a00fd6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-152.ec2.internal source: component: kubelet host: ip-10-0-129-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-da3eb} kind: Pod name: splitter-graph-da3eb-5b6db7747c-28d44 namespace: kserve-ci-e2e-test resourceVersion: "25960" uid: 2ec43c15-7101-4ecd-b712-19ef88f41b1b kind: Event lastTimestamp: "2026-09-16T19:16:46Z" message: 'Created container: splitter-graph-da3eb' metadata: creationTimestamp: "2026-09-16T19:16:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:16:46Z" name: splitter-graph-da3eb-5b6db7747c-28d44.18d5e32c61ff329a namespace: kserve-ci-e2e-test resourceVersion: "25989" uid: 2287c528-7011-4ce4-b98d-03de858cef09 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-152.ec2.internal source: component: kubelet host: ip-10-0-129-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-da3eb} kind: Pod name: splitter-graph-da3eb-5b6db7747c-28d44 namespace: kserve-ci-e2e-test resourceVersion: "25960" uid: 2ec43c15-7101-4ecd-b712-19ef88f41b1b kind: Event lastTimestamp: "2026-09-16T19:16:46Z" message: Started container splitter-graph-da3eb metadata: creationTimestamp: "2026-09-16T19:16:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:16:46Z" name: splitter-graph-da3eb-5b6db7747c-28d44.18d5e32c629362bb namespace: kserve-ci-e2e-test resourceVersion: "25990" uid: d72ba40e-640b-425f-9917-2b3744f33059 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-152.ec2.internal source: component: kubelet host: ip-10-0-129-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-da3eb} kind: Pod name: splitter-graph-da3eb-5b6db7747c-28d44 namespace: kserve-ci-e2e-test resourceVersion: "25960" uid: 2ec43c15-7101-4ecd-b712-19ef88f41b1b kind: Event lastTimestamp: "2026-09-16T19:16:56Z" message: Stopping container splitter-graph-da3eb metadata: creationTimestamp: "2026-09-16T19:16:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:16:56Z" name: splitter-graph-da3eb-5b6db7747c-28d44.18d5e32e92aa136c namespace: kserve-ci-e2e-test resourceVersion: "26374" uid: 601ed95b-6dca-4ab6-9ed0-e2e2dfa8ae7f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-152.ec2.internal source: component: kubelet host: ip-10-0-129-152.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T19:16:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-da3eb} kind: Pod name: splitter-graph-da3eb-5b6db7747c-28d44 namespace: kserve-ci-e2e-test resourceVersion: "25960" uid: 2ec43c15-7101-4ecd-b712-19ef88f41b1b kind: Event lastTimestamp: "2026-09-16T19:17:23Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-16T19:16:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:17:23Z" name: splitter-graph-da3eb-5b6db7747c-28d44.18d5e32f0c705778 namespace: kserve-ci-e2e-test resourceVersion: "26725" uid: 304539c7-1e7c-49a4-84d2-16e9ba630874 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-152.ec2.internal source: component: kubelet host: ip-10-0-129-152.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:46Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-da3eb-5b6db7747c namespace: kserve-ci-e2e-test resourceVersion: "25953" uid: 849eff4c-859e-4898-8067-deda36a9e8f9 kind: Event lastTimestamp: "2026-09-16T19:16:46Z" message: 'Created pod: splitter-graph-da3eb-5b6db7747c-28d44' metadata: creationTimestamp: "2026-09-16T19:16:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:16:46Z" name: splitter-graph-da3eb-5b6db7747c.18d5e32c3b103a24 namespace: kserve-ci-e2e-test resourceVersion: "25961" uid: ef91541d-76f1-4acb-b276-e4a707004224 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:46Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-da3eb namespace: kserve-ci-e2e-test resourceVersion: "25951" uid: 206c3b73-2941-4529-b3d8-da603edfc5bb kind: Event lastTimestamp: "2026-09-16T19:16:46Z" message: Scaled up replica set splitter-graph-da3eb-5b6db7747c from 0 to 1 metadata: creationTimestamp: "2026-09-16T19:16:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:16:46Z" name: splitter-graph-da3eb.18d5e32c3a0e2deb namespace: kserve-ci-e2e-test resourceVersion: "25956" uid: 3d2f1314-351b-4ea8-951a-bae7c162d4b4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:46Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-da3eb namespace: kserve-ci-e2e-test resourceVersion: "25970" uid: 9f90cd69-176e-4e54-8fc9-61109224fe92 kind: Event lastTimestamp: "2026-09-16T19:16:46Z" message: InferenceGraph [splitter-graph-da3eb] is Ready metadata: creationTimestamp: "2026-09-16T19:16:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:16:46Z" name: splitter-graph-da3eb.18d5e32c41aed67f namespace: kserve-ci-e2e-test resourceVersion: "25972" uid: 17bcf2f4-d81f-4158-962b-c576fa23aecf reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T18:56:45.933146Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-2796f-predictor-b7d84f855-8qlkx namespace: kserve-ci-e2e-test resourceVersion: "16135" uid: b64e0069-f0b8-4e72-a7b2-caf708191082 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-2796f-predictor-b7d84f855-8qlkx to ip-10-0-130-26.ec2.internal metadata: creationTimestamp: "2026-09-16T18:56:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T18:56:45Z" name: success-200-isvc-2796f-predictor-b7d84f855-8qlkx.18d5e214ce02c5c7 namespace: kserve-ci-e2e-test resourceVersion: "16139" uid: 80bd0718-e7a2-4bcd-b419-a904191c67bb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-594cc48779-9d5dg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:46Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-2796f-predictor-b7d84f855-8qlkx namespace: kserve-ci-e2e-test resourceVersion: "16136" uid: b64e0069-f0b8-4e72-a7b2-caf708191082 kind: Event lastTimestamp: "2026-09-16T18:56:46Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-2796f-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-16T18:56:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:46Z" name: success-200-isvc-2796f-predictor-b7d84f855-8qlkx.18d5e214d80517e6 namespace: kserve-ci-e2e-test resourceVersion: "16170" uid: 3214c721-f7df-48e6-8dfb-8c5b4e3839c4 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:47Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-2796f-predictor-b7d84f855-8qlkx namespace: kserve-ci-e2e-test resourceVersion: "16144" uid: b64e0069-f0b8-4e72-a7b2-caf708191082 kind: Event lastTimestamp: "2026-09-16T18:56:47Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T18:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T18:56:47Z" name: success-200-isvc-2796f-predictor-b7d84f855-8qlkx.18d5e21513ebdb32 namespace: kserve-ci-e2e-test resourceVersion: "16197" uid: b671eeae-b4aa-4449-8bd3-65704f5e8848 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2796f-predictor-b7d84f855-8qlkx namespace: kserve-ci-e2e-test resourceVersion: "16136" uid: b64e0069-f0b8-4e72-a7b2-caf708191082 kind: Event lastTimestamp: "2026-09-16T18:56:47Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1986" already present on machine metadata: creationTimestamp: "2026-09-16T18:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:47Z" name: success-200-isvc-2796f-predictor-b7d84f855-8qlkx.18d5e2151792cc79 namespace: kserve-ci-e2e-test resourceVersion: "16200" uid: b4b17abc-477a-4dd2-99aa-bf46df791fdc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2796f-predictor-b7d84f855-8qlkx namespace: kserve-ci-e2e-test resourceVersion: "16136" uid: b64e0069-f0b8-4e72-a7b2-caf708191082 kind: Event lastTimestamp: "2026-09-16T18:56:47Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T18:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:47Z" name: success-200-isvc-2796f-predictor-b7d84f855-8qlkx.18d5e2151b874eb6 namespace: kserve-ci-e2e-test resourceVersion: "16201" uid: 6556e0b9-49fa-4361-b455-db4b0b559ace reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2796f-predictor-b7d84f855-8qlkx namespace: kserve-ci-e2e-test resourceVersion: "16136" uid: b64e0069-f0b8-4e72-a7b2-caf708191082 kind: Event lastTimestamp: "2026-09-16T18:56:47Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T18:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:47Z" name: success-200-isvc-2796f-predictor-b7d84f855-8qlkx.18d5e2151bfa1549 namespace: kserve-ci-e2e-test resourceVersion: "16202" uid: b9cabb4d-9451-4603-819f-fbd47268b4f1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2796f-predictor-b7d84f855-8qlkx namespace: kserve-ci-e2e-test resourceVersion: "16136" uid: b64e0069-f0b8-4e72-a7b2-caf708191082 kind: Event lastTimestamp: "2026-09-16T18:56:47Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-16T18:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:47Z" name: success-200-isvc-2796f-predictor-b7d84f855-8qlkx.18d5e2151c03f224 namespace: kserve-ci-e2e-test resourceVersion: "16203" uid: 06dfba9d-e68e-4dec-a676-6be5c80888fa reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2796f-predictor-b7d84f855-8qlkx namespace: kserve-ci-e2e-test resourceVersion: "16136" uid: b64e0069-f0b8-4e72-a7b2-caf708191082 kind: Event lastTimestamp: "2026-09-16T18:56:47Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T18:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:47Z" name: success-200-isvc-2796f-predictor-b7d84f855-8qlkx.18d5e2151f159e5f namespace: kserve-ci-e2e-test resourceVersion: "16204" uid: 0ed488ac-efea-4d3f-bc66-8550f7ab907c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2796f-predictor-b7d84f855-8qlkx namespace: kserve-ci-e2e-test resourceVersion: "16136" uid: b64e0069-f0b8-4e72-a7b2-caf708191082 kind: Event lastTimestamp: "2026-09-16T18:56:47Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T18:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:47Z" name: success-200-isvc-2796f-predictor-b7d84f855-8qlkx.18d5e2151f84e106 namespace: kserve-ci-e2e-test resourceVersion: "16205" uid: 34a2f74d-62a1-4c3f-9841-ecada3b5ff02 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-16T18:56:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2796f-predictor-b7d84f855-8qlkx namespace: kserve-ci-e2e-test resourceVersion: "16136" uid: b64e0069-f0b8-4e72-a7b2-caf708191082 kind: Event lastTimestamp: "2026-09-16T19:06:03Z" message: 'Readiness probe failed: dial tcp 10.134.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T18:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:06:03Z" name: success-200-isvc-2796f-predictor-b7d84f855-8qlkx.18d5e21533ecac3e namespace: kserve-ci-e2e-test resourceVersion: "20500" uid: a8b84a7f-04b2-4af8-9c7f-54f726c975e5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2796f-predictor-b7d84f855-8qlkx namespace: kserve-ci-e2e-test resourceVersion: "16136" uid: b64e0069-f0b8-4e72-a7b2-caf708191082 kind: Event lastTimestamp: "2026-09-16T19:06:00Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T19:06:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:06:00Z" name: success-200-isvc-2796f-predictor-b7d84f855-8qlkx.18d5e295f4d3e136 namespace: kserve-ci-e2e-test resourceVersion: "20373" uid: 8fae2494-91e0-4bac-913a-5eb172050cda reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2796f-predictor-b7d84f855-8qlkx namespace: kserve-ci-e2e-test resourceVersion: "16136" uid: b64e0069-f0b8-4e72-a7b2-caf708191082 kind: Event lastTimestamp: "2026-09-16T19:06:00Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:06:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:06:00Z" name: success-200-isvc-2796f-predictor-b7d84f855-8qlkx.18d5e295f4d4704c namespace: kserve-ci-e2e-test resourceVersion: "20380" uid: c431f860-7640-4aca-951b-0e647408d825 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2796f-predictor-b7d84f855-8qlkx namespace: kserve-ci-e2e-test resourceVersion: "16136" uid: b64e0069-f0b8-4e72-a7b2-caf708191082 kind: Event lastTimestamp: "2026-09-16T19:06:03Z" message: 'Readiness probe failed: Get "https://10.134.0.30:8643/healthz": dial tcp 10.134.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:06:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:06:03Z" name: success-200-isvc-2796f-predictor-b7d84f855-8qlkx.18d5e296a866895b namespace: kserve-ci-e2e-test resourceVersion: "20499" uid: 73f2aca2-a523-42b8-927d-5beef96f25ec reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-2796f-predictor-b7d84f855 namespace: kserve-ci-e2e-test resourceVersion: "16129" uid: 552acf25-7c2c-47a5-be54-3e17f83e3660 kind: Event lastTimestamp: "2026-09-16T18:56:45Z" message: 'Created pod: success-200-isvc-2796f-predictor-b7d84f855-8qlkx' metadata: creationTimestamp: "2026-09-16T18:56:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:56:45Z" name: success-200-isvc-2796f-predictor-b7d84f855.18d5e214cd93573f namespace: kserve-ci-e2e-test resourceVersion: "16137" uid: 2d69dffb-5f21-4041-95a2-fa7b8ce7bf3b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-2796f-predictor namespace: kserve-ci-e2e-test resourceVersion: "16127" uid: ea179fa2-e099-4f36-aad6-a625ee854237 kind: Event lastTimestamp: "2026-09-16T18:56:45Z" message: Scaled up replica set success-200-isvc-2796f-predictor-b7d84f855 from 0 to 1 metadata: creationTimestamp: "2026-09-16T18:56:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:56:45Z" name: success-200-isvc-2796f-predictor.18d5e214cbdad908 namespace: kserve-ci-e2e-test resourceVersion: "16130" uid: 97c89ce7-fb09-4210-bad6-959aa2cdb2a6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T18:57:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2796f-predictor namespace: kserve-ci-e2e-test resourceVersion: "16128" uid: 0380bfa5-5c83-4bd1-aa7c-8691313ec29b kind: Event lastTimestamp: "2026-09-16T18:57:15Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T18:57:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:57:15Z" name: success-200-isvc-2796f-predictor.18d5e2184a9af0e0 namespace: kserve-ci-e2e-test resourceVersion: "16512" uid: 907e0a68-468a-44ba-b9b9-11319160e5b0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T18:57:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2796f-predictor namespace: kserve-ci-e2e-test resourceVersion: "16128" uid: 0380bfa5-5c83-4bd1-aa7c-8691313ec29b kind: Event lastTimestamp: "2026-09-16T18:57:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T18:57:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:57:15Z" name: success-200-isvc-2796f-predictor.18d5e2184b3014cc namespace: kserve-ci-e2e-test resourceVersion: "16513" uid: b20a6ed6-cf28-46f8-8e4d-8b5319b5e0cc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T18:57:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2796f-predictor namespace: kserve-ci-e2e-test resourceVersion: "16353" uid: 0380bfa5-5c83-4bd1-aa7c-8691313ec29b kind: Event lastTimestamp: "2026-09-16T18:58:01Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T18:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:58:01Z" name: success-200-isvc-2796f-predictor.18d5e21f4a5627e0 namespace: kserve-ci-e2e-test resourceVersion: "16910" uid: b596740c-cb19-4aaa-896b-2fae007356dc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T18:57:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2796f-predictor namespace: kserve-ci-e2e-test resourceVersion: "16353" uid: 0380bfa5-5c83-4bd1-aa7c-8691313ec29b kind: Event lastTimestamp: "2026-09-16T18:58:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T18:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:58:01Z" name: success-200-isvc-2796f-predictor.18d5e21f4b36fd96 namespace: kserve-ci-e2e-test resourceVersion: "16911" uid: 3858661f-0c34-49d2-9621-3088c9452825 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T18:56:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2796f namespace: kserve-ci-e2e-test resourceVersion: "16115" uid: 91a83aec-c911-413f-ac30-d5e18c1fe3b5 kind: Event lastTimestamp: "2026-09-16T18:57:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T18:56:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T18:57:33Z" name: success-200-isvc-2796f.18d5e214cbee247a namespace: kserve-ci-e2e-test resourceVersion: "16662" uid: 0022437b-1728-4c41-97ac-ed629d379f00 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2796f namespace: kserve-ci-e2e-test resourceVersion: "16660" uid: 91a83aec-c911-413f-ac30-d5e18c1fe3b5 kind: Event lastTimestamp: "2026-09-16T18:57:33Z" message: InferenceService [success-200-isvc-2796f] is Ready metadata: creationTimestamp: "2026-09-16T18:57:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T18:57:33Z" name: success-200-isvc-2796f.18d5e21fefd0e412 namespace: kserve-ci-e2e-test resourceVersion: "16661" uid: 9d92e29f-d001-45e3-8c5b-ea67dbc4ab33 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:16:56.387191Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4a90a-predictor-5c5f467b97-gbppp namespace: kserve-ci-e2e-test resourceVersion: "26430" uid: 49e28776-61dd-4e5f-8072-a0c209d00bc5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-4a90a-predictor-5c5f467b97-gbppp to ip-10-0-130-26.ec2.internal metadata: creationTimestamp: "2026-09-16T19:16:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:16:56Z" name: success-200-isvc-4a90a-predictor-5c5f467b97-gbppp.18d5e32ea2b1b9b5 namespace: kserve-ci-e2e-test resourceVersion: "26436" uid: 32330b68-994f-490a-905b-36b2d5e9463c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-594cc48779-9d5dg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:56Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4a90a-predictor-5c5f467b97-gbppp namespace: kserve-ci-e2e-test resourceVersion: "26441" uid: 49e28776-61dd-4e5f-8072-a0c209d00bc5 kind: Event lastTimestamp: "2026-09-16T19:16:56Z" message: Add eth0 [10.134.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:16:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:16:56Z" name: success-200-isvc-4a90a-predictor-5c5f467b97-gbppp.18d5e32ec500b77b namespace: kserve-ci-e2e-test resourceVersion: "26475" uid: 6288076b-00c5-4cdc-908d-28ebd2b5695d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4a90a-predictor-5c5f467b97-gbppp namespace: kserve-ci-e2e-test resourceVersion: "26434" uid: 49e28776-61dd-4e5f-8072-a0c209d00bc5 kind: Event lastTimestamp: "2026-09-16T19:16:56Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1986" already present on machine metadata: creationTimestamp: "2026-09-16T19:16:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:16:57Z" name: success-200-isvc-4a90a-predictor-5c5f467b97-gbppp.18d5e32ec727d44f namespace: kserve-ci-e2e-test resourceVersion: "26478" uid: 51e6f297-5c68-47b3-a898-2e5354a83996 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4a90a-predictor-5c5f467b97-gbppp namespace: kserve-ci-e2e-test resourceVersion: "26434" uid: 49e28776-61dd-4e5f-8072-a0c209d00bc5 kind: Event lastTimestamp: "2026-09-16T19:16:57Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T19:16:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:16:57Z" name: success-200-isvc-4a90a-predictor-5c5f467b97-gbppp.18d5e32ec9baad03 namespace: kserve-ci-e2e-test resourceVersion: "26479" uid: 6857a46a-61de-4ee8-866b-07f8d1a12fa9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4a90a-predictor-5c5f467b97-gbppp namespace: kserve-ci-e2e-test resourceVersion: "26434" uid: 49e28776-61dd-4e5f-8072-a0c209d00bc5 kind: Event lastTimestamp: "2026-09-16T19:16:57Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T19:16:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:16:57Z" name: success-200-isvc-4a90a-predictor-5c5f467b97-gbppp.18d5e32eca2ab9c7 namespace: kserve-ci-e2e-test resourceVersion: "26480" uid: eda6d7fb-c9ed-412d-8075-af8591e4d523 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4a90a-predictor-5c5f467b97-gbppp namespace: kserve-ci-e2e-test resourceVersion: "26434" uid: 49e28776-61dd-4e5f-8072-a0c209d00bc5 kind: Event lastTimestamp: "2026-09-16T19:16:57Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-16T19:16:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:16:57Z" name: success-200-isvc-4a90a-predictor-5c5f467b97-gbppp.18d5e32eca350d2e namespace: kserve-ci-e2e-test resourceVersion: "26481" uid: 3fd00bb1-0836-4a4f-909f-452f61e96f7f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4a90a-predictor-5c5f467b97-gbppp namespace: kserve-ci-e2e-test resourceVersion: "26434" uid: 49e28776-61dd-4e5f-8072-a0c209d00bc5 kind: Event lastTimestamp: "2026-09-16T19:16:57Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T19:16:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:16:57Z" name: success-200-isvc-4a90a-predictor-5c5f467b97-gbppp.18d5e32ecd657ba2 namespace: kserve-ci-e2e-test resourceVersion: "26484" uid: f088e741-83a2-4fe6-afa9-dfb033bdb130 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4a90a-predictor-5c5f467b97-gbppp namespace: kserve-ci-e2e-test resourceVersion: "26434" uid: 49e28776-61dd-4e5f-8072-a0c209d00bc5 kind: Event lastTimestamp: "2026-09-16T19:16:57Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:16:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:16:57Z" name: success-200-isvc-4a90a-predictor-5c5f467b97-gbppp.18d5e32ecdd98e57 namespace: kserve-ci-e2e-test resourceVersion: "26486" uid: fa7ab7d6-6bb2-4151-903a-707100bf557f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T19:16:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4a90a-predictor-5c5f467b97-gbppp namespace: kserve-ci-e2e-test resourceVersion: "26434" uid: 49e28776-61dd-4e5f-8072-a0c209d00bc5 kind: Event lastTimestamp: "2026-09-16T19:17:34Z" message: 'Readiness probe failed: dial tcp 10.134.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:16:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:17:34Z" name: success-200-isvc-4a90a-predictor-5c5f467b97-gbppp.18d5e32f2480e5ad namespace: kserve-ci-e2e-test resourceVersion: "26860" uid: 72c8c5f2-7be9-49c0-a182-bf1e8f21e422 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:26:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4a90a-predictor-5c5f467b97-gbppp namespace: kserve-ci-e2e-test resourceVersion: "26434" uid: 49e28776-61dd-4e5f-8072-a0c209d00bc5 kind: Event lastTimestamp: "2026-09-16T19:26:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T19:26:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:26:11Z" name: success-200-isvc-4a90a-predictor-5c5f467b97-gbppp.18d5e3afc9b0d9b8 namespace: kserve-ci-e2e-test resourceVersion: "30537" uid: 7eaa959e-2f5e-48a5-804d-a4a62df0cc7a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:26:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4a90a-predictor-5c5f467b97-gbppp namespace: kserve-ci-e2e-test resourceVersion: "26434" uid: 49e28776-61dd-4e5f-8072-a0c209d00bc5 kind: Event lastTimestamp: "2026-09-16T19:26:11Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:26:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:26:11Z" name: success-200-isvc-4a90a-predictor-5c5f467b97-gbppp.18d5e3afc9b166b9 namespace: kserve-ci-e2e-test resourceVersion: "30542" uid: 28b38fd9-65e8-4a5a-a57d-01953e8b7619 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-4a90a-predictor-5c5f467b97 namespace: kserve-ci-e2e-test resourceVersion: "26424" uid: 010a5031-9e74-400e-b2b1-1cb3b866d6b2 kind: Event lastTimestamp: "2026-09-16T19:16:56Z" message: 'Created pod: success-200-isvc-4a90a-predictor-5c5f467b97-gbppp' metadata: creationTimestamp: "2026-09-16T19:16:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:16:56Z" name: success-200-isvc-4a90a-predictor-5c5f467b97.18d5e32ea1b8fa17 namespace: kserve-ci-e2e-test resourceVersion: "26431" uid: 6e6ecf5e-43ff-47cd-855f-565262087b1c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-4a90a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26422" uid: a026713c-b4db-4eb8-a5ee-95230b5086a9 kind: Event lastTimestamp: "2026-09-16T19:16:56Z" message: Scaled up replica set success-200-isvc-4a90a-predictor-5c5f467b97 from 0 to 1 metadata: creationTimestamp: "2026-09-16T19:16:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:16:56Z" name: success-200-isvc-4a90a-predictor.18d5e32e9f653c45 namespace: kserve-ci-e2e-test resourceVersion: "26425" uid: f37ba3c2-738b-4cbd-93e9-615aed2ca99d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:17:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4a90a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26423" uid: d88a6863-e118-42f0-9a3d-887faa8ad5d7 kind: Event lastTimestamp: "2026-09-16T19:17:41Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:17:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:17:41Z" name: success-200-isvc-4a90a-predictor.18d5e3321dfc411b namespace: kserve-ci-e2e-test resourceVersion: "26918" uid: 706cda70-ca2c-4bc1-84f4-7afde8b3fb64 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:17:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4a90a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26423" uid: d88a6863-e118-42f0-9a3d-887faa8ad5d7 kind: Event lastTimestamp: "2026-09-16T19:17:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:17:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:17:41Z" name: success-200-isvc-4a90a-predictor.18d5e3321e639b6a namespace: kserve-ci-e2e-test resourceVersion: "26919" uid: 601e0500-ffb5-4f10-9599-493dedbc3d85 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T19:17:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4a90a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26618" uid: d88a6863-e118-42f0-9a3d-887faa8ad5d7 kind: Event lastTimestamp: "2026-09-16T19:18:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:17:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:18:11Z" name: success-200-isvc-4a90a-predictor.18d5e3359d1fc133 namespace: kserve-ci-e2e-test resourceVersion: "27213" uid: f59a250d-975e-4d96-a88a-04271298d398 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T19:17:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4a90a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26618" uid: d88a6863-e118-42f0-9a3d-887faa8ad5d7 kind: Event lastTimestamp: "2026-09-16T19:18:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:17:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:18:11Z" name: success-200-isvc-4a90a-predictor.18d5e3359da8afe6 namespace: kserve-ci-e2e-test resourceVersion: "27214" uid: f97930a8-4e31-4bf0-96c5-028ac8382989 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-16T19:16:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4a90a namespace: kserve-ci-e2e-test resourceVersion: "26412" uid: 91b119ae-5335-4071-929a-53c8c7675e14 kind: Event lastTimestamp: "2026-09-16T19:17:44Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T19:16:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:17:44Z" name: success-200-isvc-4a90a.18d5e32e9f77cda7 namespace: kserve-ci-e2e-test resourceVersion: "26970" uid: 44039fe2-4f17-4d74-8aad-049beede6a27 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:17:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4a90a namespace: kserve-ci-e2e-test resourceVersion: "26968" uid: 91b119ae-5335-4071-929a-53c8c7675e14 kind: Event lastTimestamp: "2026-09-16T19:17:44Z" message: InferenceService [success-200-isvc-4a90a] is Ready metadata: creationTimestamp: "2026-09-16T19:17:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:17:44Z" name: success-200-isvc-4a90a.18d5e339e054e43a namespace: kserve-ci-e2e-test resourceVersion: "26969" uid: eba21c0c-2c5f-4473-9c31-534febef45b4 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:06:31.072536Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a3448-predictor-779f587bc9-5zwl5 namespace: kserve-ci-e2e-test resourceVersion: "20859" uid: a446052c-f917-4296-8587-13e21bb06dea kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-a3448-predictor-779f587bc9-5zwl5 to ip-10-0-130-26.ec2.internal metadata: creationTimestamp: "2026-09-16T19:06:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:06:31Z" name: success-200-isvc-a3448-predictor-779f587bc9-5zwl5.18d5e29d0b09557a namespace: kserve-ci-e2e-test resourceVersion: "20864" uid: 8d57f00a-aeeb-4895-a93f-a88e1e6443e1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-594cc48779-9d5dg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:31Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a3448-predictor-779f587bc9-5zwl5 namespace: kserve-ci-e2e-test resourceVersion: "20869" uid: a446052c-f917-4296-8587-13e21bb06dea kind: Event lastTimestamp: "2026-09-16T19:06:31Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:06:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:06:31Z" name: success-200-isvc-a3448-predictor-779f587bc9-5zwl5.18d5e29d2d4f3b45 namespace: kserve-ci-e2e-test resourceVersion: "20908" uid: 76498e44-b95d-4d94-9b1e-f28986ceddd0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a3448-predictor-779f587bc9-5zwl5 namespace: kserve-ci-e2e-test resourceVersion: "20861" uid: a446052c-f917-4296-8587-13e21bb06dea kind: Event lastTimestamp: "2026-09-16T19:06:31Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1986" already present on machine metadata: creationTimestamp: "2026-09-16T19:06:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:06:31Z" name: success-200-isvc-a3448-predictor-779f587bc9-5zwl5.18d5e29d2eb66a6f namespace: kserve-ci-e2e-test resourceVersion: "20910" uid: 7f90a1d1-9efc-4cc3-927a-4b05c440d537 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a3448-predictor-779f587bc9-5zwl5 namespace: kserve-ci-e2e-test resourceVersion: "20861" uid: a446052c-f917-4296-8587-13e21bb06dea kind: Event lastTimestamp: "2026-09-16T19:06:31Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T19:06:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:06:31Z" name: success-200-isvc-a3448-predictor-779f587bc9-5zwl5.18d5e29d3165abde namespace: kserve-ci-e2e-test resourceVersion: "20911" uid: 11d7dc9d-dfc9-44f1-8231-928e77c7521e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a3448-predictor-779f587bc9-5zwl5 namespace: kserve-ci-e2e-test resourceVersion: "20861" uid: a446052c-f917-4296-8587-13e21bb06dea kind: Event lastTimestamp: "2026-09-16T19:06:31Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T19:06:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:06:31Z" name: success-200-isvc-a3448-predictor-779f587bc9-5zwl5.18d5e29d31d4203b namespace: kserve-ci-e2e-test resourceVersion: "20912" uid: 18b51635-d7b9-4fd9-bdec-6105dad5ea63 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a3448-predictor-779f587bc9-5zwl5 namespace: kserve-ci-e2e-test resourceVersion: "20861" uid: a446052c-f917-4296-8587-13e21bb06dea kind: Event lastTimestamp: "2026-09-16T19:06:31Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-16T19:06:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:06:31Z" name: success-200-isvc-a3448-predictor-779f587bc9-5zwl5.18d5e29d31dde060 namespace: kserve-ci-e2e-test resourceVersion: "20913" uid: 1baab654-7b75-4547-8062-cd362053b78a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a3448-predictor-779f587bc9-5zwl5 namespace: kserve-ci-e2e-test resourceVersion: "20861" uid: a446052c-f917-4296-8587-13e21bb06dea kind: Event lastTimestamp: "2026-09-16T19:06:31Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T19:06:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:06:31Z" name: success-200-isvc-a3448-predictor-779f587bc9-5zwl5.18d5e29d355720b6 namespace: kserve-ci-e2e-test resourceVersion: "20915" uid: 026c2deb-b529-42ad-b267-b301bc6f6550 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a3448-predictor-779f587bc9-5zwl5 namespace: kserve-ci-e2e-test resourceVersion: "20861" uid: a446052c-f917-4296-8587-13e21bb06dea kind: Event lastTimestamp: "2026-09-16T19:06:31Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:06:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:06:31Z" name: success-200-isvc-a3448-predictor-779f587bc9-5zwl5.18d5e29d35c43cba namespace: kserve-ci-e2e-test resourceVersion: "20916" uid: a212581f-d03a-4b05-934e-d5c8608d7024 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-16T19:06:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a3448-predictor-779f587bc9-5zwl5 namespace: kserve-ci-e2e-test resourceVersion: "20861" uid: a446052c-f917-4296-8587-13e21bb06dea kind: Event lastTimestamp: "2026-09-16T19:15:48Z" message: 'Readiness probe failed: dial tcp 10.134.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:06:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:15:48Z" name: success-200-isvc-a3448-predictor-779f587bc9-5zwl5.18d5e29d43b046f0 namespace: kserve-ci-e2e-test resourceVersion: "25308" uid: a383c82f-31b6-40af-91aa-625b3942edb4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:15:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a3448-predictor-779f587bc9-5zwl5 namespace: kserve-ci-e2e-test resourceVersion: "20861" uid: a446052c-f917-4296-8587-13e21bb06dea kind: Event lastTimestamp: "2026-09-16T19:15:46Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T19:15:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:15:46Z" name: success-200-isvc-a3448-predictor-779f587bc9-5zwl5.18d5e31e40793970 namespace: kserve-ci-e2e-test resourceVersion: "25213" uid: bfa9527b-2144-4476-82b5-b6d361920526 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:15:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a3448-predictor-779f587bc9-5zwl5 namespace: kserve-ci-e2e-test resourceVersion: "20861" uid: a446052c-f917-4296-8587-13e21bb06dea kind: Event lastTimestamp: "2026-09-16T19:15:46Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:15:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:15:46Z" name: success-200-isvc-a3448-predictor-779f587bc9-5zwl5.18d5e31e4079ea3f namespace: kserve-ci-e2e-test resourceVersion: "25214" uid: 9a9cd438-9520-43bf-9e96-b363fb86cb91 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:15:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a3448-predictor-779f587bc9-5zwl5 namespace: kserve-ci-e2e-test resourceVersion: "20861" uid: a446052c-f917-4296-8587-13e21bb06dea kind: Event lastTimestamp: "2026-09-16T19:15:48Z" message: 'Readiness probe failed: Get "https://10.134.0.33:8643/healthz": dial tcp 10.134.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:15:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:15:48Z" name: success-200-isvc-a3448-predictor-779f587bc9-5zwl5.18d5e31eb839a382 namespace: kserve-ci-e2e-test resourceVersion: "25307" uid: 98e9b884-6dba-4557-a5a1-c8db1365f871 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-a3448-predictor-779f587bc9 namespace: kserve-ci-e2e-test resourceVersion: "20848" uid: 2b421fd3-e4db-4473-930f-11a9b7bc1d52 kind: Event lastTimestamp: "2026-09-16T19:06:31Z" message: 'Created pod: success-200-isvc-a3448-predictor-779f587bc9-5zwl5' metadata: creationTimestamp: "2026-09-16T19:06:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:06:31Z" name: success-200-isvc-a3448-predictor-779f587bc9.18d5e29d0a7b61e9 namespace: kserve-ci-e2e-test resourceVersion: "20862" uid: d064de14-caaf-4e45-afda-69354a7186ac reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-a3448-predictor namespace: kserve-ci-e2e-test resourceVersion: "20844" uid: b367bdc1-dcd0-4c3f-b942-fb15974b42ff kind: Event lastTimestamp: "2026-09-16T19:06:31Z" message: Scaled up replica set success-200-isvc-a3448-predictor-779f587bc9 from 0 to 1 metadata: creationTimestamp: "2026-09-16T19:06:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:06:31Z" name: success-200-isvc-a3448-predictor.18d5e29d0823365c namespace: kserve-ci-e2e-test resourceVersion: "20851" uid: 4404880f-1f94-4516-899d-c7ff281a3a31 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-16T19:06:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a3448-predictor namespace: kserve-ci-e2e-test resourceVersion: "20846" uid: 1c92e50c-ac44-40c0-aa63-60e8188b29fa kind: Event lastTimestamp: "2026-09-16T19:07:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:06:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:07:46Z" name: success-200-isvc-a3448-predictor.18d5e2a0867f62db namespace: kserve-ci-e2e-test resourceVersion: "21803" uid: 0aa6f371-404a-44d2-b08d-733483471c7c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-16T19:06:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a3448-predictor namespace: kserve-ci-e2e-test resourceVersion: "20846" uid: 1c92e50c-ac44-40c0-aa63-60e8188b29fa kind: Event lastTimestamp: "2026-09-16T19:07:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:06:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:07:46Z" name: success-200-isvc-a3448-predictor.18d5e2a086e0497c namespace: kserve-ci-e2e-test resourceVersion: "21804" uid: 22416d15-a5a1-4b3a-a9ee-8fcb2bae070a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a3448-predictor namespace: kserve-ci-e2e-test resourceVersion: "21055" uid: 1c92e50c-ac44-40c0-aa63-60e8188b29fa kind: Event lastTimestamp: "2026-09-16T19:07:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:07:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:07:01Z" name: success-200-isvc-a3448-predictor.18d5e2a406f6098a namespace: kserve-ci-e2e-test resourceVersion: "21230" uid: cd413542-19d5-466e-a479-e46693c73e0e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a3448-predictor namespace: kserve-ci-e2e-test resourceVersion: "21055" uid: 1c92e50c-ac44-40c0-aa63-60e8188b29fa kind: Event lastTimestamp: "2026-09-16T19:07:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:07:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:07:01Z" name: success-200-isvc-a3448-predictor.18d5e2a4083bbcc7 namespace: kserve-ci-e2e-test resourceVersion: "21232" uid: 0dbcc087-d20a-4204-b468-75df6b77f290 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-16T19:06:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a3448 namespace: kserve-ci-e2e-test resourceVersion: "20809" uid: 6dfc16d1-0b5e-43c6-ad77-611957bc0201 kind: Event lastTimestamp: "2026-09-16T19:07:18Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T19:06:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:07:18Z" name: success-200-isvc-a3448.18d5e29d081c55a4 namespace: kserve-ci-e2e-test resourceVersion: "21528" uid: 4f8fc64a-405c-4014-a7e8-8ec12b845bd6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a3448 namespace: kserve-ci-e2e-test resourceVersion: "20809" uid: 6dfc16d1-0b5e-43c6-ad77-611957bc0201 kind: Event lastTimestamp: "2026-09-16T19:06:31Z" message: 'Failed to update status for InferenceService "success-200-isvc-a3448": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-a3448": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-16T19:06:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:06:31Z" name: success-200-isvc-a3448.18d5e29d08bd45ae namespace: kserve-ci-e2e-test resourceVersion: "20856" uid: 833a2138-9d29-48b3-ba2a-2b11b3367e07 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a3448 namespace: kserve-ci-e2e-test resourceVersion: "20809" uid: 6dfc16d1-0b5e-43c6-ad77-611957bc0201 kind: Event lastTimestamp: "2026-09-16T19:06:31Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-a3448": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-16T19:06:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:06:31Z" name: success-200-isvc-a3448.18d5e29d08bd8e81 namespace: kserve-ci-e2e-test resourceVersion: "20858" uid: e07e519a-5635-4405-baaf-b4cdeb8bf8d3 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a3448 namespace: kserve-ci-e2e-test resourceVersion: "21526" uid: 6dfc16d1-0b5e-43c6-ad77-611957bc0201 kind: Event lastTimestamp: "2026-09-16T19:07:18Z" message: InferenceService [success-200-isvc-a3448] is Ready metadata: creationTimestamp: "2026-09-16T19:07:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:07:18Z" name: success-200-isvc-a3448.18d5e2a7ff76d0ff namespace: kserve-ci-e2e-test resourceVersion: "21527" uid: 67ea9bd7-c57c-4251-a9a7-0c905959f058 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T18:54:25.775038Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-aa795-predictor-6676fd456f-w6vxm namespace: kserve-ci-e2e-test resourceVersion: "14454" uid: 31051676-c5cc-45f1-9a97-41e8e2eb8687 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-aa795-predictor-6676fd456f-w6vxm to ip-10-0-130-26.ec2.internal metadata: creationTimestamp: "2026-09-16T18:54:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T18:54:25Z" name: success-200-isvc-aa795-predictor-6676fd456f-w6vxm.18d5e1f42befca41 namespace: kserve-ci-e2e-test resourceVersion: "14460" uid: 29222ff5-a6dd-4ca2-a88f-8a7871656306 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-594cc48779-9d5dg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:54:26Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-aa795-predictor-6676fd456f-w6vxm namespace: kserve-ci-e2e-test resourceVersion: "14468" uid: 31051676-c5cc-45f1-9a97-41e8e2eb8687 kind: Event lastTimestamp: "2026-09-16T18:54:26Z" message: Add eth0 [10.134.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T18:54:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T18:54:26Z" name: success-200-isvc-aa795-predictor-6676fd456f-w6vxm.18d5e1f44e37eff8 namespace: kserve-ci-e2e-test resourceVersion: "14566" uid: 2fe3c7ec-e071-4a49-9787-fbcc5f982371 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:54:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-aa795-predictor-6676fd456f-w6vxm namespace: kserve-ci-e2e-test resourceVersion: "14457" uid: 31051676-c5cc-45f1-9a97-41e8e2eb8687 kind: Event lastTimestamp: "2026-09-16T18:54:26Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1986" metadata: creationTimestamp: "2026-09-16T18:54:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:54:26Z" name: success-200-isvc-aa795-predictor-6676fd456f-w6vxm.18d5e1f44faa3703 namespace: kserve-ci-e2e-test resourceVersion: "14568" uid: e938742f-9b66-4223-9577-3aa2d7467e58 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:54:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-aa795-predictor-6676fd456f-w6vxm namespace: kserve-ci-e2e-test resourceVersion: "14457" uid: 31051676-c5cc-45f1-9a97-41e8e2eb8687 kind: Event lastTimestamp: "2026-09-16T18:54:42Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1986" in 16.237s (16.237s including waiting). Image size: 1356771927 bytes.' metadata: creationTimestamp: "2026-09-16T18:54:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:54:42Z" name: success-200-isvc-aa795-predictor-6676fd456f-w6vxm.18d5e1f81778c261 namespace: kserve-ci-e2e-test resourceVersion: "14759" uid: 4c1a4b89-6ddf-4c7e-8e9d-0d151da37e86 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:54:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-aa795-predictor-6676fd456f-w6vxm namespace: kserve-ci-e2e-test resourceVersion: "14457" uid: 31051676-c5cc-45f1-9a97-41e8e2eb8687 kind: Event lastTimestamp: "2026-09-16T18:54:42Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T18:54:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:54:42Z" name: success-200-isvc-aa795-predictor-6676fd456f-w6vxm.18d5e1f81a8dd9f5 namespace: kserve-ci-e2e-test resourceVersion: "14760" uid: 095533d4-01b0-4321-b9e8-2fc0088bee23 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:54:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-aa795-predictor-6676fd456f-w6vxm namespace: kserve-ci-e2e-test resourceVersion: "14457" uid: 31051676-c5cc-45f1-9a97-41e8e2eb8687 kind: Event lastTimestamp: "2026-09-16T18:54:42Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T18:54:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:54:42Z" name: success-200-isvc-aa795-predictor-6676fd456f-w6vxm.18d5e1f81b14b3c8 namespace: kserve-ci-e2e-test resourceVersion: "14761" uid: 76beff0a-e374-4b64-a130-76e35780c265 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:54:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-aa795-predictor-6676fd456f-w6vxm namespace: kserve-ci-e2e-test resourceVersion: "14457" uid: 31051676-c5cc-45f1-9a97-41e8e2eb8687 kind: Event lastTimestamp: "2026-09-16T18:54:42Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-09-16T18:54:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:54:42Z" name: success-200-isvc-aa795-predictor-6676fd456f-w6vxm.18d5e1f81b22bfc2 namespace: kserve-ci-e2e-test resourceVersion: "14762" uid: 85355af0-d121-415b-9103-36955c9d82d5 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:54:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-aa795-predictor-6676fd456f-w6vxm namespace: kserve-ci-e2e-test resourceVersion: "14457" uid: 31051676-c5cc-45f1-9a97-41e8e2eb8687 kind: Event lastTimestamp: "2026-09-16T18:54:47Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 4.364s (4.364s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-09-16T18:54:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:54:47Z" name: success-200-isvc-aa795-predictor-6676fd456f-w6vxm.18d5e1f91f424844 namespace: kserve-ci-e2e-test resourceVersion: "14806" uid: 6e5de5f9-e7bf-4a78-9c5b-189f62f249fb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:54:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-aa795-predictor-6676fd456f-w6vxm namespace: kserve-ci-e2e-test resourceVersion: "14457" uid: 31051676-c5cc-45f1-9a97-41e8e2eb8687 kind: Event lastTimestamp: "2026-09-16T18:54:47Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T18:54:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:54:47Z" name: success-200-isvc-aa795-predictor-6676fd456f-w6vxm.18d5e1f924fc2ee1 namespace: kserve-ci-e2e-test resourceVersion: "14810" uid: 6beafae0-27d6-4005-83cf-79314f855014 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:54:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-aa795-predictor-6676fd456f-w6vxm namespace: kserve-ci-e2e-test resourceVersion: "14457" uid: 31051676-c5cc-45f1-9a97-41e8e2eb8687 kind: Event lastTimestamp: "2026-09-16T18:54:47Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T18:54:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:54:47Z" name: success-200-isvc-aa795-predictor-6676fd456f-w6vxm.18d5e1f925914344 namespace: kserve-ci-e2e-test resourceVersion: "14813" uid: 2d1dadf0-3f0f-4f2e-87e9-065010e4ebb7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-16T18:54:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-aa795-predictor-6676fd456f-w6vxm namespace: kserve-ci-e2e-test resourceVersion: "14457" uid: 31051676-c5cc-45f1-9a97-41e8e2eb8687 kind: Event lastTimestamp: "2026-09-16T18:55:34Z" message: 'Readiness probe failed: dial tcp 10.134.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T18:54:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:55:34Z" name: success-200-isvc-aa795-predictor-6676fd456f-w6vxm.18d5e1f95f93b958 namespace: kserve-ci-e2e-test resourceVersion: "15218" uid: 22d84ecf-b137-44e4-bf0a-01430649d9cf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-aa795-predictor-6676fd456f-w6vxm namespace: kserve-ci-e2e-test resourceVersion: "14457" uid: 31051676-c5cc-45f1-9a97-41e8e2eb8687 kind: Event lastTimestamp: "2026-09-16T18:56:05Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T18:56:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:05Z" name: success-200-isvc-aa795-predictor-6676fd456f-w6vxm.18d5e20b772ae6af namespace: kserve-ci-e2e-test resourceVersion: "15556" uid: 329c5724-4afc-4874-ba0c-9d0f439ccbef reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-aa795-predictor-6676fd456f-w6vxm namespace: kserve-ci-e2e-test resourceVersion: "14457" uid: 31051676-c5cc-45f1-9a97-41e8e2eb8687 kind: Event lastTimestamp: "2026-09-16T18:56:05Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T18:56:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:05Z" name: success-200-isvc-aa795-predictor-6676fd456f-w6vxm.18d5e20b772c1543 namespace: kserve-ci-e2e-test resourceVersion: "15562" uid: 4fe84522-7e45-4f3d-a4ce-86640510c098 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:54:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-aa795-predictor-6676fd456f namespace: kserve-ci-e2e-test resourceVersion: "14447" uid: a1329356-74c0-49d8-a6be-fffe584f14cb kind: Event lastTimestamp: "2026-09-16T18:54:25Z" message: 'Created pod: success-200-isvc-aa795-predictor-6676fd456f-w6vxm' metadata: creationTimestamp: "2026-09-16T18:54:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:54:25Z" name: success-200-isvc-aa795-predictor-6676fd456f.18d5e1f42b57c07a namespace: kserve-ci-e2e-test resourceVersion: "14456" uid: ea79cbc5-4ff0-426f-a212-32691f28f512 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:54:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-aa795-predictor namespace: kserve-ci-e2e-test resourceVersion: "14445" uid: c9fabe37-f403-425f-bf7b-2b7c780a91ec kind: Event lastTimestamp: "2026-09-16T18:54:25Z" message: Scaled up replica set success-200-isvc-aa795-predictor-6676fd456f from 0 to 1 metadata: creationTimestamp: "2026-09-16T18:54:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:54:25Z" name: success-200-isvc-aa795-predictor.18d5e1f4298f5618 namespace: kserve-ci-e2e-test resourceVersion: "14448" uid: bebb1ae7-79a2-4afc-bc5f-655a1d5d6b15 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T18:54:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-aa795-predictor namespace: kserve-ci-e2e-test resourceVersion: "14446" uid: 5c77d80f-0871-46c8-8d54-0c33c1e64eb4 kind: Event lastTimestamp: "2026-09-16T18:54:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T18:54:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:54:55Z" name: success-200-isvc-aa795-predictor.18d5e1f7a80e258a namespace: kserve-ci-e2e-test resourceVersion: "14888" uid: 60d1d7e0-57b1-40cf-90cb-84f7e436933e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T18:54:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-aa795-predictor namespace: kserve-ci-e2e-test resourceVersion: "14446" uid: 5c77d80f-0871-46c8-8d54-0c33c1e64eb4 kind: Event lastTimestamp: "2026-09-16T18:54:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T18:54:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:54:55Z" name: success-200-isvc-aa795-predictor.18d5e1f7a88c7434 namespace: kserve-ci-e2e-test resourceVersion: "14889" uid: f21ed4a4-2b4d-463d-be40-7035dc43ac18 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-16T18:55:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-aa795-predictor namespace: kserve-ci-e2e-test resourceVersion: "14733" uid: 5c77d80f-0871-46c8-8d54-0c33c1e64eb4 kind: Event lastTimestamp: "2026-09-16T18:55:55Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T18:55:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:55:55Z" name: success-200-isvc-aa795-predictor.18d5e1fea5fd4a88 namespace: kserve-ci-e2e-test resourceVersion: "15421" uid: 91a45c96-80ff-4505-a09f-0b4bb54b26aa reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-16T18:55:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-aa795-predictor namespace: kserve-ci-e2e-test resourceVersion: "14733" uid: 5c77d80f-0871-46c8-8d54-0c33c1e64eb4 kind: Event lastTimestamp: "2026-09-16T18:55:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T18:55:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:55:55Z" name: success-200-isvc-aa795-predictor.18d5e1fea66ba65c namespace: kserve-ci-e2e-test resourceVersion: "15423" uid: 6c782807-b114-448b-84f2-7a5ed4948ae5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-16T18:54:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-aa795 namespace: kserve-ci-e2e-test resourceVersion: "14437" uid: 8188ddf4-107d-470c-9bd8-8d570a8a91a9 kind: Event lastTimestamp: "2026-09-16T18:55:44Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T18:54:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T18:55:44Z" name: success-200-isvc-aa795.18d5e1f429993fb1 namespace: kserve-ci-e2e-test resourceVersion: "15311" uid: 069280b9-1e24-49c5-866a-7b027f16399c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:55:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-aa795 namespace: kserve-ci-e2e-test resourceVersion: "15308" uid: 8188ddf4-107d-470c-9bd8-8d570a8a91a9 kind: Event lastTimestamp: "2026-09-16T18:55:44Z" message: InferenceService [success-200-isvc-aa795] is Ready metadata: creationTimestamp: "2026-09-16T18:55:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T18:55:44Z" name: success-200-isvc-aa795.18d5e2066f367dc3 namespace: kserve-ci-e2e-test resourceVersion: "15310" uid: d1de312d-92f4-459a-ae10-7c35f83367eb reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:05:20.881538Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c6de0-predictor-654f67bf9f-7f7cx namespace: kserve-ci-e2e-test resourceVersion: "19946" uid: 44f9db22-17eb-42e7-8d88-1c26d286ef85 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-c6de0-predictor-654f67bf9f-7f7cx to ip-10-0-130-26.ec2.internal metadata: creationTimestamp: "2026-09-16T19:05:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:05:20Z" name: success-200-isvc-c6de0-predictor-654f67bf9f-7f7cx.18d5e28cb353bd04 namespace: kserve-ci-e2e-test resourceVersion: "19959" uid: 4c8df8e4-5afd-44ab-88cb-88cbd3a01cdd reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-594cc48779-9d5dg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:05:21Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c6de0-predictor-654f67bf9f-7f7cx namespace: kserve-ci-e2e-test resourceVersion: "19967" uid: 44f9db22-17eb-42e7-8d88-1c26d286ef85 kind: Event lastTimestamp: "2026-09-16T19:05:21Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:05:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:05:21Z" name: success-200-isvc-c6de0-predictor-654f67bf9f-7f7cx.18d5e28cd5aea5a7 namespace: kserve-ci-e2e-test resourceVersion: "20009" uid: b46bc2f1-0c01-4c6d-a04b-317fb0c52317 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:05:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c6de0-predictor-654f67bf9f-7f7cx namespace: kserve-ci-e2e-test resourceVersion: "19953" uid: 44f9db22-17eb-42e7-8d88-1c26d286ef85 kind: Event lastTimestamp: "2026-09-16T19:05:21Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1986" already present on machine metadata: creationTimestamp: "2026-09-16T19:05:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:05:21Z" name: success-200-isvc-c6de0-predictor-654f67bf9f-7f7cx.18d5e28cd7477484 namespace: kserve-ci-e2e-test resourceVersion: "20013" uid: fd1406ad-f050-4168-bc7f-65c6b0f3b774 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:05:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c6de0-predictor-654f67bf9f-7f7cx namespace: kserve-ci-e2e-test resourceVersion: "19953" uid: 44f9db22-17eb-42e7-8d88-1c26d286ef85 kind: Event lastTimestamp: "2026-09-16T19:05:21Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T19:05:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:05:21Z" name: success-200-isvc-c6de0-predictor-654f67bf9f-7f7cx.18d5e28cda427fe6 namespace: kserve-ci-e2e-test resourceVersion: "20015" uid: d3a62daa-7737-47fe-b934-fa25f7df0d6c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:05:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c6de0-predictor-654f67bf9f-7f7cx namespace: kserve-ci-e2e-test resourceVersion: "19953" uid: 44f9db22-17eb-42e7-8d88-1c26d286ef85 kind: Event lastTimestamp: "2026-09-16T19:05:21Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T19:05:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:05:21Z" name: success-200-isvc-c6de0-predictor-654f67bf9f-7f7cx.18d5e28cdab72c8d namespace: kserve-ci-e2e-test resourceVersion: "20016" uid: c4d19632-fd82-4c25-bb57-908a7d1478be reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:05:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c6de0-predictor-654f67bf9f-7f7cx namespace: kserve-ci-e2e-test resourceVersion: "19953" uid: 44f9db22-17eb-42e7-8d88-1c26d286ef85 kind: Event lastTimestamp: "2026-09-16T19:05:21Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-16T19:05:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:05:21Z" name: success-200-isvc-c6de0-predictor-654f67bf9f-7f7cx.18d5e28cdac11f55 namespace: kserve-ci-e2e-test resourceVersion: "20017" uid: 7a4706b5-4438-45e7-bbcf-6cff2cc20a30 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:05:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c6de0-predictor-654f67bf9f-7f7cx namespace: kserve-ci-e2e-test resourceVersion: "19953" uid: 44f9db22-17eb-42e7-8d88-1c26d286ef85 kind: Event lastTimestamp: "2026-09-16T19:05:21Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T19:05:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:05:21Z" name: success-200-isvc-c6de0-predictor-654f67bf9f-7f7cx.18d5e28cddd543b1 namespace: kserve-ci-e2e-test resourceVersion: "20018" uid: 9eeef395-3ac2-4571-b2e0-1cc5aef15a5f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:05:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c6de0-predictor-654f67bf9f-7f7cx namespace: kserve-ci-e2e-test resourceVersion: "19953" uid: 44f9db22-17eb-42e7-8d88-1c26d286ef85 kind: Event lastTimestamp: "2026-09-16T19:05:21Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:05:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:05:21Z" name: success-200-isvc-c6de0-predictor-654f67bf9f-7f7cx.18d5e28cde456b9a namespace: kserve-ci-e2e-test resourceVersion: "20019" uid: 3c9de906-d80e-45bb-9f7a-eb61ba2e1ed4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T19:05:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c6de0-predictor-654f67bf9f-7f7cx namespace: kserve-ci-e2e-test resourceVersion: "19953" uid: 44f9db22-17eb-42e7-8d88-1c26d286ef85 kind: Event lastTimestamp: "2026-09-16T19:05:57Z" message: 'Readiness probe failed: dial tcp 10.134.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:05:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:05:57Z" name: success-200-isvc-c6de0-predictor-654f67bf9f-7f7cx.18d5e28ce4e94863 namespace: kserve-ci-e2e-test resourceVersion: "20316" uid: 611bdc59-0984-45c9-bd5a-75e56cf721b3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c6de0-predictor-654f67bf9f-7f7cx namespace: kserve-ci-e2e-test resourceVersion: "19953" uid: 44f9db22-17eb-42e7-8d88-1c26d286ef85 kind: Event lastTimestamp: "2026-09-16T19:06:30Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T19:06:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:06:30Z" name: success-200-isvc-c6de0-predictor-654f67bf9f-7f7cx.18d5e29d0452f91b namespace: kserve-ci-e2e-test resourceVersion: "20824" uid: 97a0595e-7792-4993-a961-3e4774801c83 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c6de0-predictor-654f67bf9f-7f7cx namespace: kserve-ci-e2e-test resourceVersion: "19953" uid: 44f9db22-17eb-42e7-8d88-1c26d286ef85 kind: Event lastTimestamp: "2026-09-16T19:06:30Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:06:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:06:30Z" name: success-200-isvc-c6de0-predictor-654f67bf9f-7f7cx.18d5e29d045392fe namespace: kserve-ci-e2e-test resourceVersion: "20828" uid: d6ff80be-a8a5-46b1-9397-fb35a83417c2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c6de0-predictor-654f67bf9f-7f7cx namespace: kserve-ci-e2e-test resourceVersion: "19953" uid: 44f9db22-17eb-42e7-8d88-1c26d286ef85 kind: Event lastTimestamp: "2026-09-16T19:06:32Z" message: 'Readiness probe failed: Get "https://10.134.0.31:8643/healthz": dial tcp 10.134.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:06:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:06:32Z" name: success-200-isvc-c6de0-predictor-654f67bf9f-7f7cx.18d5e29d6d2f5705 namespace: kserve-ci-e2e-test resourceVersion: "20939" uid: 523e2026-93ae-441a-96ed-2e0c71670532 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:05:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-c6de0-predictor-654f67bf9f namespace: kserve-ci-e2e-test resourceVersion: "19922" uid: 6cdfb944-4a1a-42e8-9448-bf132ea88660 kind: Event lastTimestamp: "2026-09-16T19:05:20Z" message: 'Created pod: success-200-isvc-c6de0-predictor-654f67bf9f-7f7cx' metadata: creationTimestamp: "2026-09-16T19:05:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:05:20Z" name: success-200-isvc-c6de0-predictor-654f67bf9f.18d5e28cb171d194 namespace: kserve-ci-e2e-test resourceVersion: "19952" uid: 59bb3f84-7f14-4047-99fa-93f4e8e51f90 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:05:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-c6de0-predictor namespace: kserve-ci-e2e-test resourceVersion: "19918" uid: 06f40701-0052-4ed3-89a3-be40e3d2cd37 kind: Event lastTimestamp: "2026-09-16T19:05:20Z" message: Scaled up replica set success-200-isvc-c6de0-predictor-654f67bf9f from 0 to 1 metadata: creationTimestamp: "2026-09-16T19:05:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:05:20Z" name: success-200-isvc-c6de0-predictor.18d5e28ca8720873 namespace: kserve-ci-e2e-test resourceVersion: "19923" uid: 22a29be7-ad5f-407a-954b-4ce905536d37 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:05:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c6de0-predictor namespace: kserve-ci-e2e-test resourceVersion: "19921" uid: 6706b3bd-69b0-4827-a9a3-3e42dd61a98d kind: Event lastTimestamp: "2026-09-16T19:05:50Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:05:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:05:50Z" name: success-200-isvc-c6de0-predictor.18d5e29026eeaa59 namespace: kserve-ci-e2e-test resourceVersion: "20261" uid: 37975524-274b-4524-964b-295924b6a08b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:05:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c6de0-predictor namespace: kserve-ci-e2e-test resourceVersion: "19921" uid: 6706b3bd-69b0-4827-a9a3-3e42dd61a98d kind: Event lastTimestamp: "2026-09-16T19:05:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:05:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:05:50Z" name: success-200-isvc-c6de0-predictor.18d5e2902757c09e namespace: kserve-ci-e2e-test resourceVersion: "20262" uid: 4261ee5b-ee95-4cea-bc27-0181316c7866 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:06:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c6de0-predictor namespace: kserve-ci-e2e-test resourceVersion: "20149" uid: 6706b3bd-69b0-4827-a9a3-3e42dd61a98d kind: Event lastTimestamp: "2026-09-16T19:06:20Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:06:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:06:20Z" name: success-200-isvc-c6de0-predictor.18d5e29724eb1326 namespace: kserve-ci-e2e-test resourceVersion: "20679" uid: 30e765c4-916b-4e60-83e3-cb7400803566 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:06:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c6de0-predictor namespace: kserve-ci-e2e-test resourceVersion: "20149" uid: 6706b3bd-69b0-4827-a9a3-3e42dd61a98d kind: Event lastTimestamp: "2026-09-16T19:06:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:06:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:06:20Z" name: success-200-isvc-c6de0-predictor.18d5e297258c70c0 namespace: kserve-ci-e2e-test resourceVersion: "20682" uid: b3baed74-a1c1-4989-be41-d2b402b49d0a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-16T19:05:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c6de0 namespace: kserve-ci-e2e-test resourceVersion: "19912" uid: b5525ded-04fd-4478-827b-610db52bc478 kind: Event lastTimestamp: "2026-09-16T19:06:07Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T19:05:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:06:07Z" name: success-200-isvc-c6de0.18d5e28ca8b7aa15 namespace: kserve-ci-e2e-test resourceVersion: "20549" uid: 2d8c581c-fcb8-4caf-9f04-a232a05dc140 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c6de0 namespace: kserve-ci-e2e-test resourceVersion: "20546" uid: b5525ded-04fd-4478-827b-610db52bc478 kind: Event lastTimestamp: "2026-09-16T19:06:07Z" message: InferenceService [success-200-isvc-c6de0] is Ready metadata: creationTimestamp: "2026-09-16T19:06:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:06:07Z" name: success-200-isvc-c6de0.18d5e297a06faa43 namespace: kserve-ci-e2e-test resourceVersion: "20548" uid: af22f4ad-77c1-4cb8-9d6e-b890460e7056 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:16:25.923209Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ca22e-predictor-558b7cd765-n4t6k namespace: kserve-ci-e2e-test resourceVersion: "25687" uid: ef314c3b-f862-4b89-8a16-633b13991399 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-ca22e-predictor-558b7cd765-n4t6k to ip-10-0-130-26.ec2.internal metadata: creationTimestamp: "2026-09-16T19:16:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:16:25Z" name: success-200-isvc-ca22e-predictor-558b7cd765-n4t6k.18d5e3278ae63e88 namespace: kserve-ci-e2e-test resourceVersion: "25692" uid: e8ad0157-97c1-4364-bd2e-3392c2356c1d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-594cc48779-9d5dg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:26Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ca22e-predictor-558b7cd765-n4t6k namespace: kserve-ci-e2e-test resourceVersion: "25689" uid: ef314c3b-f862-4b89-8a16-633b13991399 kind: Event lastTimestamp: "2026-09-16T19:16:26Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-ca22e-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-16T19:16:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:16:26Z" name: success-200-isvc-ca22e-predictor-558b7cd765-n4t6k.18d5e32791659c62 namespace: kserve-ci-e2e-test resourceVersion: "25711" uid: 8764ba52-1e55-447d-9b96-9166670c4afc reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:27Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ca22e-predictor-558b7cd765-n4t6k namespace: kserve-ci-e2e-test resourceVersion: "25702" uid: ef314c3b-f862-4b89-8a16-633b13991399 kind: Event lastTimestamp: "2026-09-16T19:16:27Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:16:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:16:27Z" name: success-200-isvc-ca22e-predictor-558b7cd765-n4t6k.18d5e327d0c96bf2 namespace: kserve-ci-e2e-test resourceVersion: "25740" uid: 825cb15c-ddc3-49d9-8ea6-aec792a1af04 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ca22e-predictor-558b7cd765-n4t6k namespace: kserve-ci-e2e-test resourceVersion: "25689" uid: ef314c3b-f862-4b89-8a16-633b13991399 kind: Event lastTimestamp: "2026-09-16T19:16:27Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1986" already present on machine metadata: creationTimestamp: "2026-09-16T19:16:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:16:27Z" name: success-200-isvc-ca22e-predictor-558b7cd765-n4t6k.18d5e327d298edd6 namespace: kserve-ci-e2e-test resourceVersion: "25743" uid: 867d687f-0667-42d9-aeda-1a106a551d6e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ca22e-predictor-558b7cd765-n4t6k namespace: kserve-ci-e2e-test resourceVersion: "25689" uid: ef314c3b-f862-4b89-8a16-633b13991399 kind: Event lastTimestamp: "2026-09-16T19:16:27Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T19:16:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:16:27Z" name: success-200-isvc-ca22e-predictor-558b7cd765-n4t6k.18d5e327d59acdaa namespace: kserve-ci-e2e-test resourceVersion: "25744" uid: c7e8deb2-1bed-48ab-8b5f-848be56dfbaa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ca22e-predictor-558b7cd765-n4t6k namespace: kserve-ci-e2e-test resourceVersion: "25689" uid: ef314c3b-f862-4b89-8a16-633b13991399 kind: Event lastTimestamp: "2026-09-16T19:16:27Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T19:16:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:16:27Z" name: success-200-isvc-ca22e-predictor-558b7cd765-n4t6k.18d5e327d60cadb0 namespace: kserve-ci-e2e-test resourceVersion: "25745" uid: 64506a15-da70-4594-b1f9-2167acfa56ff reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ca22e-predictor-558b7cd765-n4t6k namespace: kserve-ci-e2e-test resourceVersion: "25689" uid: ef314c3b-f862-4b89-8a16-633b13991399 kind: Event lastTimestamp: "2026-09-16T19:16:27Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-16T19:16:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:16:27Z" name: success-200-isvc-ca22e-predictor-558b7cd765-n4t6k.18d5e327d6168463 namespace: kserve-ci-e2e-test resourceVersion: "25746" uid: 50b50e62-d7ca-4b2e-bb80-1721751c96cd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ca22e-predictor-558b7cd765-n4t6k namespace: kserve-ci-e2e-test resourceVersion: "25689" uid: ef314c3b-f862-4b89-8a16-633b13991399 kind: Event lastTimestamp: "2026-09-16T19:16:27Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T19:16:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:16:27Z" name: success-200-isvc-ca22e-predictor-558b7cd765-n4t6k.18d5e327dab77c29 namespace: kserve-ci-e2e-test resourceVersion: "25751" uid: 4ee74403-8ab3-41f5-864a-bbeb7a0a7787 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ca22e-predictor-558b7cd765-n4t6k namespace: kserve-ci-e2e-test resourceVersion: "25689" uid: ef314c3b-f862-4b89-8a16-633b13991399 kind: Event lastTimestamp: "2026-09-16T19:16:27Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:16:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:16:27Z" name: success-200-isvc-ca22e-predictor-558b7cd765-n4t6k.18d5e327db2c8420 namespace: kserve-ci-e2e-test resourceVersion: "25752" uid: e04f5bdb-7c70-4f5e-be76-303820301912 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T19:16:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ca22e-predictor-558b7cd765-n4t6k namespace: kserve-ci-e2e-test resourceVersion: "25689" uid: ef314c3b-f862-4b89-8a16-633b13991399 kind: Event lastTimestamp: "2026-09-16T19:17:04Z" message: 'Readiness probe failed: dial tcp 10.134.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:16:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:17:04Z" name: success-200-isvc-ca22e-predictor-558b7cd765-n4t6k.18d5e32820c7b700 namespace: kserve-ci-e2e-test resourceVersion: "26559" uid: 6281e68f-e806-4b55-b3d9-d24cb0d0e3ca reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:33:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ca22e-predictor-558b7cd765-n4t6k namespace: kserve-ci-e2e-test resourceVersion: "25689" uid: ef314c3b-f862-4b89-8a16-633b13991399 kind: Event lastTimestamp: "2026-09-16T19:33:45Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T19:33:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:33:45Z" name: success-200-isvc-ca22e-predictor-558b7cd765-n4t6k.18d5e4198bac87e2 namespace: kserve-ci-e2e-test resourceVersion: "33678" uid: 2c54ce8c-bd7b-4466-8b01-efb0e94b328e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:33:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ca22e-predictor-558b7cd765-n4t6k namespace: kserve-ci-e2e-test resourceVersion: "25689" uid: ef314c3b-f862-4b89-8a16-633b13991399 kind: Event lastTimestamp: "2026-09-16T19:33:45Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:33:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:33:45Z" name: success-200-isvc-ca22e-predictor-558b7cd765-n4t6k.18d5e4198bad98c1 namespace: kserve-ci-e2e-test resourceVersion: "33679" uid: be2c4e77-dab6-4244-8133-50ccd7ef3b3a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:33:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ca22e-predictor-558b7cd765-n4t6k namespace: kserve-ci-e2e-test resourceVersion: "25689" uid: ef314c3b-f862-4b89-8a16-633b13991399 kind: Event lastTimestamp: "2026-09-16T19:33:49Z" message: 'Readiness probe failed: Get "https://10.134.0.36:8643/healthz": dial tcp 10.134.0.36:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:33:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:33:49Z" name: success-200-isvc-ca22e-predictor-558b7cd765-n4t6k.18d5e41a817d030d namespace: kserve-ci-e2e-test resourceVersion: "33763" uid: 4d9e7e57-520e-41da-aa79-4ae9c75f9ee1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:33:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ca22e-predictor-558b7cd765-n4t6k namespace: kserve-ci-e2e-test resourceVersion: "25689" uid: ef314c3b-f862-4b89-8a16-633b13991399 kind: Event lastTimestamp: "2026-09-16T19:33:55Z" message: 'Readiness probe failed: Get "https://10.134.0.36:8643/healthz": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)' metadata: creationTimestamp: "2026-09-16T19:33:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:33:55Z" name: success-200-isvc-ca22e-predictor-558b7cd765-n4t6k.18d5e41be71aa940 namespace: kserve-ci-e2e-test resourceVersion: "33811" uid: abc03d64-a6fa-491c-bf79-b0b15adf6567 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:33:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ca22e-predictor-558b7cd765-n4t6k namespace: kserve-ci-e2e-test resourceVersion: "25689" uid: ef314c3b-f862-4b89-8a16-633b13991399 kind: Event lastTimestamp: "2026-09-16T19:33:55Z" message: 'Readiness probe failed: dial tcp 10.134.0.36:8080: i/o timeout' metadata: creationTimestamp: "2026-09-16T19:33:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:33:55Z" name: success-200-isvc-ca22e-predictor-558b7cd765-n4t6k.18d5e41be75bb9e2 namespace: kserve-ci-e2e-test resourceVersion: "33812" uid: 2d336b56-80dd-48ea-8bf1-598f9a77439a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-ca22e-predictor-558b7cd765 namespace: kserve-ci-e2e-test resourceVersion: "25679" uid: 04129299-047a-479d-bf80-33966fa36b10 kind: Event lastTimestamp: "2026-09-16T19:16:25Z" message: 'Created pod: success-200-isvc-ca22e-predictor-558b7cd765-n4t6k' metadata: creationTimestamp: "2026-09-16T19:16:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:16:25Z" name: success-200-isvc-ca22e-predictor-558b7cd765.18d5e3278a3f353e namespace: kserve-ci-e2e-test resourceVersion: "25690" uid: db3bb0a7-9ae1-4f16-b233-dfb929591c0b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-ca22e-predictor namespace: kserve-ci-e2e-test resourceVersion: "25676" uid: c0e39697-9c38-414a-b6ea-7111f0269d87 kind: Event lastTimestamp: "2026-09-16T19:16:25Z" message: Scaled up replica set success-200-isvc-ca22e-predictor-558b7cd765 from 0 to 1 metadata: creationTimestamp: "2026-09-16T19:16:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:16:25Z" name: success-200-isvc-ca22e-predictor.18d5e32787411c1d namespace: kserve-ci-e2e-test resourceVersion: "25680" uid: d4a03d67-218d-4ab3-81e7-3a43859589a2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:16:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ca22e-predictor namespace: kserve-ci-e2e-test resourceVersion: "25678" uid: cdad4db2-f710-46a3-b71d-aa1632f8a010 kind: Event lastTimestamp: "2026-09-16T19:16:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:16:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:16:55Z" name: success-200-isvc-ca22e-predictor.18d5e32b0600890f namespace: kserve-ci-e2e-test resourceVersion: "26353" uid: feae888f-f02a-4334-8c7e-812e8081a7c3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:16:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ca22e-predictor namespace: kserve-ci-e2e-test resourceVersion: "25678" uid: cdad4db2-f710-46a3-b71d-aa1632f8a010 kind: Event lastTimestamp: "2026-09-16T19:16:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:16:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:16:55Z" name: success-200-isvc-ca22e-predictor.18d5e32b064f0c80 namespace: kserve-ci-e2e-test resourceVersion: "26354" uid: 3e702a4b-210f-479d-a89c-b1a70b8f6856 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-16T19:17:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ca22e-predictor namespace: kserve-ci-e2e-test resourceVersion: "25899" uid: cdad4db2-f710-46a3-b71d-aa1632f8a010 kind: Event lastTimestamp: "2026-09-16T19:17:55Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:17:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:17:55Z" name: success-200-isvc-ca22e-predictor.18d5e33203eeb8e4 namespace: kserve-ci-e2e-test resourceVersion: "27050" uid: 7661440f-af9f-481d-b1a5-8955c2e162b4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-16T19:17:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ca22e-predictor namespace: kserve-ci-e2e-test resourceVersion: "25899" uid: cdad4db2-f710-46a3-b71d-aa1632f8a010 kind: Event lastTimestamp: "2026-09-16T19:17:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:17:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:17:55Z" name: success-200-isvc-ca22e-predictor.18d5e33204651f67 namespace: kserve-ci-e2e-test resourceVersion: "27051" uid: ebb52911-ba4e-4b40-ab15-9f841d9636ef reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-16T19:16:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ca22e namespace: kserve-ci-e2e-test resourceVersion: "25657" uid: d78dc4c3-0197-472f-98c1-f9d87187dbef kind: Event lastTimestamp: "2026-09-16T19:17:14Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T19:16:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:17:14Z" name: success-200-isvc-ca22e.18d5e327875a0e56 namespace: kserve-ci-e2e-test resourceVersion: "26656" uid: 84d9b18f-d4a4-486b-93ca-a3e9d6c8eeb9 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ca22e namespace: kserve-ci-e2e-test resourceVersion: "25657" uid: d78dc4c3-0197-472f-98c1-f9d87187dbef kind: Event lastTimestamp: "2026-09-16T19:16:25Z" message: 'Failed to update status for InferenceService "success-200-isvc-ca22e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-ca22e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-16T19:16:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:16:25Z" name: success-200-isvc-ca22e.18d5e32787e66d55 namespace: kserve-ci-e2e-test resourceVersion: "25683" uid: cc9fa890-175d-4fe2-9e12-7bc1feedf744 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ca22e namespace: kserve-ci-e2e-test resourceVersion: "25657" uid: d78dc4c3-0197-472f-98c1-f9d87187dbef kind: Event lastTimestamp: "2026-09-16T19:16:25Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-ca22e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-16T19:16:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:16:25Z" name: success-200-isvc-ca22e.18d5e32787e6aa86 namespace: kserve-ci-e2e-test resourceVersion: "25684" uid: 14e6e1aa-32d7-4922-99dd-0fed697e1e63 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:17:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ca22e namespace: kserve-ci-e2e-test resourceVersion: "26654" uid: d78dc4c3-0197-472f-98c1-f9d87187dbef kind: Event lastTimestamp: "2026-09-16T19:17:14Z" message: InferenceService [success-200-isvc-ca22e] is Ready metadata: creationTimestamp: "2026-09-16T19:17:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:17:14Z" name: success-200-isvc-ca22e.18d5e332dc336657 namespace: kserve-ci-e2e-test resourceVersion: "26655" uid: fe9a018c-5104-443d-bef5-3886b6f2e7e8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:15:46.058620Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-da3eb-predictor-5d58fc86fc-fchjf namespace: kserve-ci-e2e-test resourceVersion: "25215" uid: d4457dd2-6c6b-42d4-b9f9-1cb5e2f35d0e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-da3eb-predictor-5d58fc86fc-fchjf to ip-10-0-130-26.ec2.internal metadata: creationTimestamp: "2026-09-16T19:15:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:15:46Z" name: success-200-isvc-da3eb-predictor-5d58fc86fc-fchjf.18d5e31e42c8f6b0 namespace: kserve-ci-e2e-test resourceVersion: "25228" uid: 751155fc-8f96-4b7e-8f77-d911fe3cf7db reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-594cc48779-9d5dg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:15:46Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-da3eb-predictor-5d58fc86fc-fchjf namespace: kserve-ci-e2e-test resourceVersion: "25235" uid: d4457dd2-6c6b-42d4-b9f9-1cb5e2f35d0e kind: Event lastTimestamp: "2026-09-16T19:15:46Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:15:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:15:46Z" name: success-200-isvc-da3eb-predictor-5d58fc86fc-fchjf.18d5e31e64c3adbe namespace: kserve-ci-e2e-test resourceVersion: "25278" uid: f9532e32-4a72-47d0-9be3-5ece4a2bfbd3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:15:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-da3eb-predictor-5d58fc86fc-fchjf namespace: kserve-ci-e2e-test resourceVersion: "25218" uid: d4457dd2-6c6b-42d4-b9f9-1cb5e2f35d0e kind: Event lastTimestamp: "2026-09-16T19:15:46Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1986" already present on machine metadata: creationTimestamp: "2026-09-16T19:15:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:15:46Z" name: success-200-isvc-da3eb-predictor-5d58fc86fc-fchjf.18d5e31e66da0b33 namespace: kserve-ci-e2e-test resourceVersion: "25280" uid: 729c5782-d672-4a11-95d0-c22c00827995 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:15:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-da3eb-predictor-5d58fc86fc-fchjf namespace: kserve-ci-e2e-test resourceVersion: "25218" uid: d4457dd2-6c6b-42d4-b9f9-1cb5e2f35d0e kind: Event lastTimestamp: "2026-09-16T19:15:46Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T19:15:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:15:46Z" name: success-200-isvc-da3eb-predictor-5d58fc86fc-fchjf.18d5e31e69725fc2 namespace: kserve-ci-e2e-test resourceVersion: "25282" uid: 8109fb1a-f9f3-4af4-80c8-b43932bf4a68 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:15:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-da3eb-predictor-5d58fc86fc-fchjf namespace: kserve-ci-e2e-test resourceVersion: "25218" uid: d4457dd2-6c6b-42d4-b9f9-1cb5e2f35d0e kind: Event lastTimestamp: "2026-09-16T19:15:46Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T19:15:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:15:46Z" name: success-200-isvc-da3eb-predictor-5d58fc86fc-fchjf.18d5e31e69f4c110 namespace: kserve-ci-e2e-test resourceVersion: "25284" uid: 6727ffa7-8d6c-4da7-bd30-ac7977c6db52 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:15:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-da3eb-predictor-5d58fc86fc-fchjf namespace: kserve-ci-e2e-test resourceVersion: "25218" uid: d4457dd2-6c6b-42d4-b9f9-1cb5e2f35d0e kind: Event lastTimestamp: "2026-09-16T19:15:46Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-16T19:15:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:15:46Z" name: success-200-isvc-da3eb-predictor-5d58fc86fc-fchjf.18d5e31e69fe080b namespace: kserve-ci-e2e-test resourceVersion: "25285" uid: 00038d5c-2b04-4943-b2a4-407d3512b64a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:15:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-da3eb-predictor-5d58fc86fc-fchjf namespace: kserve-ci-e2e-test resourceVersion: "25218" uid: d4457dd2-6c6b-42d4-b9f9-1cb5e2f35d0e kind: Event lastTimestamp: "2026-09-16T19:15:46Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T19:15:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:15:46Z" name: success-200-isvc-da3eb-predictor-5d58fc86fc-fchjf.18d5e31e6ced5af7 namespace: kserve-ci-e2e-test resourceVersion: "25286" uid: 4867610f-decd-4b62-89cb-6551be18f154 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:15:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-da3eb-predictor-5d58fc86fc-fchjf namespace: kserve-ci-e2e-test resourceVersion: "25218" uid: d4457dd2-6c6b-42d4-b9f9-1cb5e2f35d0e kind: Event lastTimestamp: "2026-09-16T19:15:46Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:15:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:15:46Z" name: success-200-isvc-da3eb-predictor-5d58fc86fc-fchjf.18d5e31e6d5d2ad2 namespace: kserve-ci-e2e-test resourceVersion: "25288" uid: 0e1e430b-9633-4389-8613-ab69c9c7bd33 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T19:15:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-da3eb-predictor-5d58fc86fc-fchjf namespace: kserve-ci-e2e-test resourceVersion: "25218" uid: d4457dd2-6c6b-42d4-b9f9-1cb5e2f35d0e kind: Event lastTimestamp: "2026-09-16T19:16:24Z" message: 'Readiness probe failed: dial tcp 10.134.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:15:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:16:24Z" name: success-200-isvc-da3eb-predictor-5d58fc86fc-fchjf.18d5e31ec6779ed0 namespace: kserve-ci-e2e-test resourceVersion: "25603" uid: 7fb53fe1-86e7-449c-8105-398e70644db8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-da3eb-predictor-5d58fc86fc-fchjf namespace: kserve-ci-e2e-test resourceVersion: "25218" uid: d4457dd2-6c6b-42d4-b9f9-1cb5e2f35d0e kind: Event lastTimestamp: "2026-09-16T19:16:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T19:16:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:16:56Z" name: success-200-isvc-da3eb-predictor-5d58fc86fc-fchjf.18d5e32e9992ba3c namespace: kserve-ci-e2e-test resourceVersion: "26397" uid: f78d63d8-188a-4fbb-97f9-2c7ee4d9c0d7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-da3eb-predictor-5d58fc86fc-fchjf namespace: kserve-ci-e2e-test resourceVersion: "25218" uid: d4457dd2-6c6b-42d4-b9f9-1cb5e2f35d0e kind: Event lastTimestamp: "2026-09-16T19:16:56Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:16:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:16:56Z" name: success-200-isvc-da3eb-predictor-5d58fc86fc-fchjf.18d5e32e9993e200 namespace: kserve-ci-e2e-test resourceVersion: "26404" uid: e0c88d06-ade8-4520-aebe-ec12e3d2ddb9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-da3eb-predictor-5d58fc86fc-fchjf namespace: kserve-ci-e2e-test resourceVersion: "25218" uid: d4457dd2-6c6b-42d4-b9f9-1cb5e2f35d0e kind: Event lastTimestamp: "2026-09-16T19:16:59Z" message: 'Readiness probe failed: Get "https://10.134.0.35:8643/healthz": dial tcp 10.134.0.35:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:16:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:16:59Z" name: success-200-isvc-da3eb-predictor-5d58fc86fc-fchjf.18d5e32f4ec47308 namespace: kserve-ci-e2e-test resourceVersion: "26516" uid: 0781639e-9732-44c1-9831-348a122136f2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:15:46Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-da3eb-predictor-5d58fc86fc namespace: kserve-ci-e2e-test resourceVersion: "25192" uid: 53a55aee-3fd5-4c25-abdd-6b6768108dbc kind: Event lastTimestamp: "2026-09-16T19:15:46Z" message: 'Created pod: success-200-isvc-da3eb-predictor-5d58fc86fc-fchjf' metadata: creationTimestamp: "2026-09-16T19:15:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:15:46Z" name: success-200-isvc-da3eb-predictor-5d58fc86fc.18d5e31e41df3e14 namespace: kserve-ci-e2e-test resourceVersion: "25217" uid: 94fd2eea-0d49-4d13-90f5-1a916c361a79 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:15:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-da3eb-predictor namespace: kserve-ci-e2e-test resourceVersion: "25190" uid: 094de410-9227-4548-a392-e0d6b22b1ca9 kind: Event lastTimestamp: "2026-09-16T19:15:45Z" message: Scaled up replica set success-200-isvc-da3eb-predictor-5d58fc86fc from 0 to 1 metadata: creationTimestamp: "2026-09-16T19:15:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:15:45Z" name: success-200-isvc-da3eb-predictor.18d5e31e3cb783ac namespace: kserve-ci-e2e-test resourceVersion: "25194" uid: 5bfb6154-256e-484e-bd87-740ac8e2259f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:16:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-da3eb-predictor namespace: kserve-ci-e2e-test resourceVersion: "25191" uid: 979ab4fa-085f-43cf-ab95-ad49714f99ab kind: Event lastTimestamp: "2026-09-16T19:16:15Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:16:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:16:15Z" name: success-200-isvc-da3eb-predictor.18d5e321bb52e564 namespace: kserve-ci-e2e-test resourceVersion: "25539" uid: f1b96ab7-6192-40ea-a827-00fb6e5123d1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:16:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-da3eb-predictor namespace: kserve-ci-e2e-test resourceVersion: "25191" uid: 979ab4fa-085f-43cf-ab95-ad49714f99ab kind: Event lastTimestamp: "2026-09-16T19:16:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:16:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:16:16Z" name: success-200-isvc-da3eb-predictor.18d5e321bbc7ebc9 namespace: kserve-ci-e2e-test resourceVersion: "25540" uid: ae7b2b02-77f6-4612-8126-979f4a0324ba reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:16:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-da3eb-predictor namespace: kserve-ci-e2e-test resourceVersion: "25423" uid: 979ab4fa-085f-43cf-ab95-ad49714f99ab kind: Event lastTimestamp: "2026-09-16T19:16:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:16:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:16:46Z" name: success-200-isvc-da3eb-predictor.18d5e328b9db0fcd namespace: kserve-ci-e2e-test resourceVersion: "25955" uid: f2513ab3-fa61-4303-b295-f2aa7928822f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:16:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-da3eb-predictor namespace: kserve-ci-e2e-test resourceVersion: "25423" uid: 979ab4fa-085f-43cf-ab95-ad49714f99ab kind: Event lastTimestamp: "2026-09-16T19:16:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:16:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:16:46Z" name: success-200-isvc-da3eb-predictor.18d5e328ba37647a namespace: kserve-ci-e2e-test resourceVersion: "25958" uid: 9a01192c-8c89-444c-8916-be8707a46924 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-16T19:15:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-da3eb namespace: kserve-ci-e2e-test resourceVersion: "25182" uid: e6aa007d-1d7e-47c4-a697-f88abb11ba48 kind: Event lastTimestamp: "2026-09-16T19:16:34Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T19:15:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:16:34Z" name: success-200-isvc-da3eb.18d5e31e3cb7761c namespace: kserve-ci-e2e-test resourceVersion: "25843" uid: 80afd680-ea2f-4754-8573-e0103d5ed6c0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:15:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-da3eb namespace: kserve-ci-e2e-test resourceVersion: "25182" uid: e6aa007d-1d7e-47c4-a697-f88abb11ba48 kind: Event lastTimestamp: "2026-09-16T19:15:45Z" message: 'Failed to update status for InferenceService "success-200-isvc-da3eb": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-da3eb": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-16T19:15:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:15:45Z" name: success-200-isvc-da3eb.18d5e31e3d5237b0 namespace: kserve-ci-e2e-test resourceVersion: "25197" uid: c2b9bfb3-42a0-4283-9695-fe367006ef32 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:15:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-da3eb namespace: kserve-ci-e2e-test resourceVersion: "25182" uid: e6aa007d-1d7e-47c4-a697-f88abb11ba48 kind: Event lastTimestamp: "2026-09-16T19:15:45Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-da3eb": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-16T19:15:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:15:45Z" name: success-200-isvc-da3eb.18d5e31e3d5289d2 namespace: kserve-ci-e2e-test resourceVersion: "25203" uid: 2b1f2eae-6724-4e76-a539-5a2c0949dd14 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-da3eb namespace: kserve-ci-e2e-test resourceVersion: "25841" uid: e6aa007d-1d7e-47c4-a697-f88abb11ba48 kind: Event lastTimestamp: "2026-09-16T19:16:34Z" message: InferenceService [success-200-isvc-da3eb] is Ready metadata: creationTimestamp: "2026-09-16T19:16:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:16:34Z" name: success-200-isvc-da3eb.18d5e329822fdbb9 namespace: kserve-ci-e2e-test resourceVersion: "25842" uid: c4831996-464f-4d71-b7f3-666396ddbceb reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:07:11.015956Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e99c4-predictor-5dcbf77d7f-hqcjf namespace: kserve-ci-e2e-test resourceVersion: "21384" uid: df26cccb-22ae-4076-8ad2-e9218f84d64d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-e99c4-predictor-5dcbf77d7f-hqcjf to ip-10-0-130-26.ec2.internal metadata: creationTimestamp: "2026-09-16T19:07:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:07:11Z" name: success-200-isvc-e99c4-predictor-5dcbf77d7f-hqcjf.18d5e2a657d99368 namespace: kserve-ci-e2e-test resourceVersion: "21390" uid: 39e96304-715c-4d3e-a330-e9f0a451d41d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-594cc48779-9d5dg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:11Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e99c4-predictor-5dcbf77d7f-hqcjf namespace: kserve-ci-e2e-test resourceVersion: "21388" uid: df26cccb-22ae-4076-8ad2-e9218f84d64d kind: Event lastTimestamp: "2026-09-16T19:07:11Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-e99c4-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-16T19:07:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:11Z" name: success-200-isvc-e99c4-predictor-5dcbf77d7f-hqcjf.18d5e2a661e67feb namespace: kserve-ci-e2e-test resourceVersion: "21426" uid: 5c28d0e5-a9b6-45da-8167-00dee74e9546 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:12Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e99c4-predictor-5dcbf77d7f-hqcjf namespace: kserve-ci-e2e-test resourceVersion: "21394" uid: df26cccb-22ae-4076-8ad2-e9218f84d64d kind: Event lastTimestamp: "2026-09-16T19:07:12Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:07:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:07:12Z" name: success-200-isvc-e99c4-predictor-5dcbf77d7f-hqcjf.18d5e2a69db925ad namespace: kserve-ci-e2e-test resourceVersion: "21447" uid: 1b29e5eb-ebab-4e01-8edc-879c9ea4e4f8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e99c4-predictor-5dcbf77d7f-hqcjf namespace: kserve-ci-e2e-test resourceVersion: "21388" uid: df26cccb-22ae-4076-8ad2-e9218f84d64d kind: Event lastTimestamp: "2026-09-16T19:07:12Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1986" already present on machine metadata: creationTimestamp: "2026-09-16T19:07:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:12Z" name: success-200-isvc-e99c4-predictor-5dcbf77d7f-hqcjf.18d5e2a69f1bd269 namespace: kserve-ci-e2e-test resourceVersion: "21450" uid: 8ddedb86-80a6-4350-872d-57084019a58d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e99c4-predictor-5dcbf77d7f-hqcjf namespace: kserve-ci-e2e-test resourceVersion: "21388" uid: df26cccb-22ae-4076-8ad2-e9218f84d64d kind: Event lastTimestamp: "2026-09-16T19:07:12Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T19:07:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:12Z" name: success-200-isvc-e99c4-predictor-5dcbf77d7f-hqcjf.18d5e2a6a1bbb8a4 namespace: kserve-ci-e2e-test resourceVersion: "21452" uid: b7ecfb40-27fb-44ea-856a-63890aef1fc2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e99c4-predictor-5dcbf77d7f-hqcjf namespace: kserve-ci-e2e-test resourceVersion: "21388" uid: df26cccb-22ae-4076-8ad2-e9218f84d64d kind: Event lastTimestamp: "2026-09-16T19:07:12Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T19:07:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:12Z" name: success-200-isvc-e99c4-predictor-5dcbf77d7f-hqcjf.18d5e2a6a22faf8f namespace: kserve-ci-e2e-test resourceVersion: "21453" uid: db8d77ba-d33b-4ce2-9a7b-92771ee1f2bf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e99c4-predictor-5dcbf77d7f-hqcjf namespace: kserve-ci-e2e-test resourceVersion: "21388" uid: df26cccb-22ae-4076-8ad2-e9218f84d64d kind: Event lastTimestamp: "2026-09-16T19:07:12Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-16T19:07:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:12Z" name: success-200-isvc-e99c4-predictor-5dcbf77d7f-hqcjf.18d5e2a6a23b47fc namespace: kserve-ci-e2e-test resourceVersion: "21454" uid: 4c861ef0-b7eb-4c61-a5bd-85a54fbfaaef reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e99c4-predictor-5dcbf77d7f-hqcjf namespace: kserve-ci-e2e-test resourceVersion: "21388" uid: df26cccb-22ae-4076-8ad2-e9218f84d64d kind: Event lastTimestamp: "2026-09-16T19:07:12Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T19:07:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:12Z" name: success-200-isvc-e99c4-predictor-5dcbf77d7f-hqcjf.18d5e2a6a5057487 namespace: kserve-ci-e2e-test resourceVersion: "21456" uid: 43e2bccc-3d0d-4274-a307-592e6956b818 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e99c4-predictor-5dcbf77d7f-hqcjf namespace: kserve-ci-e2e-test resourceVersion: "21388" uid: df26cccb-22ae-4076-8ad2-e9218f84d64d kind: Event lastTimestamp: "2026-09-16T19:07:12Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:07:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:12Z" name: success-200-isvc-e99c4-predictor-5dcbf77d7f-hqcjf.18d5e2a6a5744914 namespace: kserve-ci-e2e-test resourceVersion: "21457" uid: 02097c2c-1757-4e34-8680-65c17ec707e7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-16T19:07:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e99c4-predictor-5dcbf77d7f-hqcjf namespace: kserve-ci-e2e-test resourceVersion: "21388" uid: df26cccb-22ae-4076-8ad2-e9218f84d64d kind: Event lastTimestamp: "2026-09-16T19:16:29Z" message: 'Readiness probe failed: dial tcp 10.134.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:07:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:16:29Z" name: success-200-isvc-e99c4-predictor-5dcbf77d7f-hqcjf.18d5e2a6da1c1b3a namespace: kserve-ci-e2e-test resourceVersion: "25781" uid: 85fb83af-f588-426f-9502-e21186038daa reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e99c4-predictor-5dcbf77d7f-hqcjf namespace: kserve-ci-e2e-test resourceVersion: "21388" uid: df26cccb-22ae-4076-8ad2-e9218f84d64d kind: Event lastTimestamp: "2026-09-16T19:16:25Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T19:16:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:16:25Z" name: success-200-isvc-e99c4-predictor-5dcbf77d7f-hqcjf.18d5e3278340a8b0 namespace: kserve-ci-e2e-test resourceVersion: "25665" uid: 54b05013-d22b-428b-9ea6-a24119447cfe reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e99c4-predictor-5dcbf77d7f-hqcjf namespace: kserve-ci-e2e-test resourceVersion: "21388" uid: df26cccb-22ae-4076-8ad2-e9218f84d64d kind: Event lastTimestamp: "2026-09-16T19:16:25Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:16:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:16:25Z" name: success-200-isvc-e99c4-predictor-5dcbf77d7f-hqcjf.18d5e3278341462b namespace: kserve-ci-e2e-test resourceVersion: "25669" uid: e4ebcebe-5b93-4b97-b7b2-26590816ae2a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:16:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e99c4-predictor-5dcbf77d7f-hqcjf namespace: kserve-ci-e2e-test resourceVersion: "21388" uid: df26cccb-22ae-4076-8ad2-e9218f84d64d kind: Event lastTimestamp: "2026-09-16T19:16:29Z" message: 'Readiness probe failed: Get "https://10.134.0.34:8643/healthz": dial tcp 10.134.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:16:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:16:29Z" name: success-200-isvc-e99c4-predictor-5dcbf77d7f-hqcjf.18d5e3284e903d1f namespace: kserve-ci-e2e-test resourceVersion: "25780" uid: 059f4253-5b20-4060-a8c2-4b71e3dcc9e3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-e99c4-predictor-5dcbf77d7f namespace: kserve-ci-e2e-test resourceVersion: "21371" uid: cf1bc91d-316f-4ef8-97a9-1fef915a4e9c kind: Event lastTimestamp: "2026-09-16T19:07:11Z" message: 'Created pod: success-200-isvc-e99c4-predictor-5dcbf77d7f-hqcjf' metadata: creationTimestamp: "2026-09-16T19:07:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:07:11Z" name: success-200-isvc-e99c4-predictor-5dcbf77d7f.18d5e2a65762b5c1 namespace: kserve-ci-e2e-test resourceVersion: "21387" uid: 514222e9-8d17-44da-bea1-de029aa4e7f5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-e99c4-predictor namespace: kserve-ci-e2e-test resourceVersion: "21363" uid: 9ade1102-b2fc-4457-94bf-2e006e7b904c kind: Event lastTimestamp: "2026-09-16T19:07:10Z" message: Scaled up replica set success-200-isvc-e99c4-predictor-5dcbf77d7f from 0 to 1 metadata: creationTimestamp: "2026-09-16T19:07:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:07:10Z" name: success-200-isvc-e99c4-predictor.18d5e2a654f191d2 namespace: kserve-ci-e2e-test resourceVersion: "21374" uid: 75023131-08b1-483f-904a-c74f28c9cb74 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T19:07:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e99c4-predictor namespace: kserve-ci-e2e-test resourceVersion: "21367" uid: 4fabed3e-bfad-4a04-acc3-de8faed25086 kind: Event lastTimestamp: "2026-09-16T19:07:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:07:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:07:56Z" name: success-200-isvc-e99c4-predictor.18d5e2a9d333a0fd namespace: kserve-ci-e2e-test resourceVersion: "21877" uid: 61facdd9-b65a-4cec-b86d-0bee2ed24593 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T19:07:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e99c4-predictor namespace: kserve-ci-e2e-test resourceVersion: "21367" uid: 4fabed3e-bfad-4a04-acc3-de8faed25086 kind: Event lastTimestamp: "2026-09-16T19:07:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:07:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:07:56Z" name: success-200-isvc-e99c4-predictor.18d5e2a9d39f5135 namespace: kserve-ci-e2e-test resourceVersion: "21878" uid: b9bde0f1-afa1-428b-991f-67713303d7fc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T19:08:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e99c4-predictor namespace: kserve-ci-e2e-test resourceVersion: "21601" uid: 4fabed3e-bfad-4a04-acc3-de8faed25086 kind: Event lastTimestamp: "2026-09-16T19:08:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:08:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:08:41Z" name: success-200-isvc-e99c4-predictor.18d5e2b4524f7575 namespace: kserve-ci-e2e-test resourceVersion: "22263" uid: 1d8a74ef-44c5-40c0-9d93-86b27b03dc24 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T19:08:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e99c4-predictor namespace: kserve-ci-e2e-test resourceVersion: "21601" uid: 4fabed3e-bfad-4a04-acc3-de8faed25086 kind: Event lastTimestamp: "2026-09-16T19:08:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:08:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:08:41Z" name: success-200-isvc-e99c4-predictor.18d5e2b45373fc9c namespace: kserve-ci-e2e-test resourceVersion: "22264" uid: 4b78900a-25c4-4eeb-9387-8dd52b93c735 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T19:07:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e99c4 namespace: kserve-ci-e2e-test resourceVersion: "21336" uid: c90d2557-3d61-4c7e-b79c-c982db7b1b89 kind: Event lastTimestamp: "2026-09-16T19:07:59Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T19:07:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:07:59Z" name: success-200-isvc-e99c4.18d5e2a654ef9422 namespace: kserve-ci-e2e-test resourceVersion: "21919" uid: 65ca8b48-2f76-4444-88ad-ef3988682ae6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e99c4 namespace: kserve-ci-e2e-test resourceVersion: "21336" uid: c90d2557-3d61-4c7e-b79c-c982db7b1b89 kind: Event lastTimestamp: "2026-09-16T19:07:10Z" message: 'Failed to update status for InferenceService "success-200-isvc-e99c4": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-e99c4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-16T19:07:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:07:10Z" name: success-200-isvc-e99c4.18d5e2a655874a33 namespace: kserve-ci-e2e-test resourceVersion: "21377" uid: 99836012-f596-4699-b89c-64cf966407d0 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e99c4 namespace: kserve-ci-e2e-test resourceVersion: "21336" uid: c90d2557-3d61-4c7e-b79c-c982db7b1b89 kind: Event lastTimestamp: "2026-09-16T19:07:10Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-e99c4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-16T19:07:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:07:10Z" name: success-200-isvc-e99c4.18d5e2a6558794be namespace: kserve-ci-e2e-test resourceVersion: "21379" uid: 6dfc5fcb-1d9f-4fe8-bd86-5497c7a9e3d8 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e99c4 namespace: kserve-ci-e2e-test resourceVersion: "21917" uid: c90d2557-3d61-4c7e-b79c-c982db7b1b89 kind: Event lastTimestamp: "2026-09-16T19:07:59Z" message: InferenceService [success-200-isvc-e99c4] is Ready metadata: creationTimestamp: "2026-09-16T19:07:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:07:59Z" name: success-200-isvc-e99c4.18d5e2b195ef4a02 namespace: kserve-ci-e2e-test resourceVersion: "21918" uid: 1a6b364c-b8a0-46f3-89cc-519f6a8fc513 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T18:56:05.970273Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-eb09c-predictor-555759ccc-6z7w5 namespace: kserve-ci-e2e-test resourceVersion: "15588" uid: 42ce6703-df67-490f-90dc-671f7e90ce29 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-eb09c-predictor-555759ccc-6z7w5 to ip-10-0-130-26.ec2.internal metadata: creationTimestamp: "2026-09-16T18:56:05Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T18:56:05Z" name: success-200-isvc-eb09c-predictor-555759ccc-6z7w5.18d5e20b8009c5f1 namespace: kserve-ci-e2e-test resourceVersion: "15592" uid: 43eca2b0-d62b-4995-9927-51fd233a6e2c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-594cc48779-9d5dg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:06Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-eb09c-predictor-555759ccc-6z7w5 namespace: kserve-ci-e2e-test resourceVersion: "15605" uid: 42ce6703-df67-490f-90dc-671f7e90ce29 kind: Event lastTimestamp: "2026-09-16T18:56:06Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T18:56:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T18:56:06Z" name: success-200-isvc-eb09c-predictor-555759ccc-6z7w5.18d5e20ba2099920 namespace: kserve-ci-e2e-test resourceVersion: "15639" uid: 01022a16-62c5-43b6-b368-2d292bbae08c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-eb09c-predictor-555759ccc-6z7w5 namespace: kserve-ci-e2e-test resourceVersion: "15590" uid: 42ce6703-df67-490f-90dc-671f7e90ce29 kind: Event lastTimestamp: "2026-09-16T18:56:06Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1986" already present on machine metadata: creationTimestamp: "2026-09-16T18:56:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:06Z" name: success-200-isvc-eb09c-predictor-555759ccc-6z7w5.18d5e20ba3bf8e07 namespace: kserve-ci-e2e-test resourceVersion: "15643" uid: 827b1b67-218e-403f-908c-673445d16614 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-eb09c-predictor-555759ccc-6z7w5 namespace: kserve-ci-e2e-test resourceVersion: "15590" uid: 42ce6703-df67-490f-90dc-671f7e90ce29 kind: Event lastTimestamp: "2026-09-16T18:56:06Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T18:56:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:06Z" name: success-200-isvc-eb09c-predictor-555759ccc-6z7w5.18d5e20ba66f47c0 namespace: kserve-ci-e2e-test resourceVersion: "15647" uid: 5f6769ea-098c-4653-aa3c-a7fc171b0c70 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-eb09c-predictor-555759ccc-6z7w5 namespace: kserve-ci-e2e-test resourceVersion: "15590" uid: 42ce6703-df67-490f-90dc-671f7e90ce29 kind: Event lastTimestamp: "2026-09-16T18:56:06Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T18:56:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:06Z" name: success-200-isvc-eb09c-predictor-555759ccc-6z7w5.18d5e20ba6dc1830 namespace: kserve-ci-e2e-test resourceVersion: "15649" uid: 4f53d594-8629-482b-80e1-28a38b3b0e6e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-eb09c-predictor-555759ccc-6z7w5 namespace: kserve-ci-e2e-test resourceVersion: "15590" uid: 42ce6703-df67-490f-90dc-671f7e90ce29 kind: Event lastTimestamp: "2026-09-16T18:56:06Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-16T18:56:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:06Z" name: success-200-isvc-eb09c-predictor-555759ccc-6z7w5.18d5e20ba6e5fd7f namespace: kserve-ci-e2e-test resourceVersion: "15651" uid: 8b9f2852-0664-46d4-802d-3441c0b084c5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-eb09c-predictor-555759ccc-6z7w5 namespace: kserve-ci-e2e-test resourceVersion: "15590" uid: 42ce6703-df67-490f-90dc-671f7e90ce29 kind: Event lastTimestamp: "2026-09-16T18:56:06Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T18:56:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:06Z" name: success-200-isvc-eb09c-predictor-555759ccc-6z7w5.18d5e20ba99d44b6 namespace: kserve-ci-e2e-test resourceVersion: "15653" uid: c5fd0416-d280-421b-9221-f8424682d8c2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-eb09c-predictor-555759ccc-6z7w5 namespace: kserve-ci-e2e-test resourceVersion: "15590" uid: 42ce6703-df67-490f-90dc-671f7e90ce29 kind: Event lastTimestamp: "2026-09-16T18:56:06Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T18:56:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:06Z" name: success-200-isvc-eb09c-predictor-555759ccc-6z7w5.18d5e20baa0ad617 namespace: kserve-ci-e2e-test resourceVersion: "15654" uid: 22ff9b52-ed8d-47d4-ad02-5d09bf2a885f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-16T18:56:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-eb09c-predictor-555759ccc-6z7w5 namespace: kserve-ci-e2e-test resourceVersion: "15590" uid: 42ce6703-df67-490f-90dc-671f7e90ce29 kind: Event lastTimestamp: "2026-09-16T19:05:23Z" message: 'Readiness probe failed: dial tcp 10.134.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T18:56:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:05:23Z" name: success-200-isvc-eb09c-predictor-555759ccc-6z7w5.18d5e20bd88661c0 namespace: kserve-ci-e2e-test resourceVersion: "20047" uid: 31d3c181-1ff5-4864-8659-199a01332aba reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:05:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-eb09c-predictor-555759ccc-6z7w5 namespace: kserve-ci-e2e-test resourceVersion: "15590" uid: 42ce6703-df67-490f-90dc-671f7e90ce29 kind: Event lastTimestamp: "2026-09-16T19:05:20Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T19:05:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:05:20Z" name: success-200-isvc-eb09c-predictor-555759ccc-6z7w5.18d5e28cadcefc37 namespace: kserve-ci-e2e-test resourceVersion: "19941" uid: e5b5bd87-e06d-4b9a-8936-38821688eda7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:05:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-eb09c-predictor-555759ccc-6z7w5 namespace: kserve-ci-e2e-test resourceVersion: "15590" uid: 42ce6703-df67-490f-90dc-671f7e90ce29 kind: Event lastTimestamp: "2026-09-16T19:05:20Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:05:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:05:20Z" name: success-200-isvc-eb09c-predictor-555759ccc-6z7w5.18d5e28cadcf7adb namespace: kserve-ci-e2e-test resourceVersion: "19942" uid: bd658a29-74e1-4f8d-87d1-6940b84b6836 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:05:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-eb09c-predictor-555759ccc-6z7w5 namespace: kserve-ci-e2e-test resourceVersion: "15590" uid: 42ce6703-df67-490f-90dc-671f7e90ce29 kind: Event lastTimestamp: "2026-09-16T19:05:23Z" message: 'Readiness probe failed: Get "https://10.134.0.29:8643/healthz": dial tcp 10.134.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:05:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:05:23Z" name: success-200-isvc-eb09c-predictor-555759ccc-6z7w5.18d5e28d4d11478d namespace: kserve-ci-e2e-test resourceVersion: "20046" uid: 42f3e1f1-676f-40b7-b527-6dbff0842840 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:05Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-eb09c-predictor-555759ccc namespace: kserve-ci-e2e-test resourceVersion: "15581" uid: e25c800a-9ddd-45bf-8cf6-7e6e272f7080 kind: Event lastTimestamp: "2026-09-16T18:56:05Z" message: 'Created pod: success-200-isvc-eb09c-predictor-555759ccc-6z7w5' metadata: creationTimestamp: "2026-09-16T18:56:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:56:05Z" name: success-200-isvc-eb09c-predictor-555759ccc.18d5e20b7f78bd10 namespace: kserve-ci-e2e-test resourceVersion: "15589" uid: 4263e8a4-3a23-40a7-abca-6c49f126219c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:05Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-eb09c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15579" uid: 2f27c786-c73a-41c6-b8e3-37c8d04f4673 kind: Event lastTimestamp: "2026-09-16T18:56:05Z" message: Scaled up replica set success-200-isvc-eb09c-predictor-555759ccc from 0 to 1 metadata: creationTimestamp: "2026-09-16T18:56:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:56:05Z" name: success-200-isvc-eb09c-predictor.18d5e20b7c7f6826 namespace: kserve-ci-e2e-test resourceVersion: "15584" uid: f0def2f7-33eb-42b4-bed6-27ca5c130185 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T18:56:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-eb09c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15580" uid: 5df6c345-f35e-49b6-9670-7054306737cd kind: Event lastTimestamp: "2026-09-16T18:56:35Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T18:56:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:56:35Z" name: success-200-isvc-eb09c-predictor.18d5e20efb5cce1f namespace: kserve-ci-e2e-test resourceVersion: "15954" uid: 9d4b19f5-640e-4999-a3c1-7c073c234004 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T18:56:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-eb09c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15580" uid: 5df6c345-f35e-49b6-9670-7054306737cd kind: Event lastTimestamp: "2026-09-16T18:56:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T18:56:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:56:35Z" name: success-200-isvc-eb09c-predictor.18d5e20efbc70614 namespace: kserve-ci-e2e-test resourceVersion: "15955" uid: d61f22cb-a5d6-4b44-8291-73fdc2ca49e2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T18:56:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-eb09c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15806" uid: 5df6c345-f35e-49b6-9670-7054306737cd kind: Event lastTimestamp: "2026-09-16T18:57:20Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T18:56:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:57:20Z" name: success-200-isvc-eb09c-predictor.18d5e215f944cc2e namespace: kserve-ci-e2e-test resourceVersion: "16558" uid: ff59b59a-aaec-42fc-b23a-14008674bb26 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T18:56:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-eb09c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15806" uid: 5df6c345-f35e-49b6-9670-7054306737cd kind: Event lastTimestamp: "2026-09-16T18:57:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T18:56:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:57:20Z" name: success-200-isvc-eb09c-predictor.18d5e215f9b43110 namespace: kserve-ci-e2e-test resourceVersion: "16559" uid: 6f981412-8e3c-4a56-9414-8bf3da8a8ef2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-16T18:56:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-eb09c namespace: kserve-ci-e2e-test resourceVersion: "15570" uid: 4d76bead-e773-46f2-ac03-80924dee9da1 kind: Event lastTimestamp: "2026-09-16T18:56:53Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T18:56:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T18:56:53Z" name: success-200-isvc-eb09c.18d5e20b7c7f258b namespace: kserve-ci-e2e-test resourceVersion: "16288" uid: 773b2c75-4f80-4843-a587-c9756777b88e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-eb09c namespace: kserve-ci-e2e-test resourceVersion: "16286" uid: 4d76bead-e773-46f2-ac03-80924dee9da1 kind: Event lastTimestamp: "2026-09-16T18:56:53Z" message: InferenceService [success-200-isvc-eb09c] is Ready metadata: creationTimestamp: "2026-09-16T18:56:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T18:56:53Z" name: success-200-isvc-eb09c.18d5e21693e513f1 namespace: kserve-ci-e2e-test resourceVersion: "16287" uid: ce7e6df9-29be-468e-bb0f-36a336f16c6e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:06:00.750161Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f64b9-predictor-ffc69556f-g6c6n namespace: kserve-ci-e2e-test resourceVersion: "20407" uid: 73c06bb6-1195-4d3a-9a58-d01702b61f73 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-f64b9-predictor-ffc69556f-g6c6n to ip-10-0-130-26.ec2.internal metadata: creationTimestamp: "2026-09-16T19:06:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:06:00Z" name: success-200-isvc-f64b9-predictor-ffc69556f-g6c6n.18d5e295fbae8e0c namespace: kserve-ci-e2e-test resourceVersion: "20411" uid: 2a0ba343-56ba-4c00-a068-4f2b04cae083 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-594cc48779-9d5dg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:00Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f64b9-predictor-ffc69556f-g6c6n namespace: kserve-ci-e2e-test resourceVersion: "20410" uid: 73c06bb6-1195-4d3a-9a58-d01702b61f73 kind: Event lastTimestamp: "2026-09-16T19:06:00Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-f64b9-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-16T19:06:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:06:00Z" name: success-200-isvc-f64b9-predictor-ffc69556f-g6c6n.18d5e2960a32aee4 namespace: kserve-ci-e2e-test resourceVersion: "20442" uid: 51c3deaf-13e0-4274-b7c8-a03406ee3744 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:01Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f64b9-predictor-ffc69556f-g6c6n namespace: kserve-ci-e2e-test resourceVersion: "20416" uid: 73c06bb6-1195-4d3a-9a58-d01702b61f73 kind: Event lastTimestamp: "2026-09-16T19:06:01Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:06:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:06:01Z" name: success-200-isvc-f64b9-predictor-ffc69556f-g6c6n.18d5e296417c149c namespace: kserve-ci-e2e-test resourceVersion: "20470" uid: 3e804064-b3b9-4f2a-a756-4818521a8fc6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f64b9-predictor-ffc69556f-g6c6n namespace: kserve-ci-e2e-test resourceVersion: "20410" uid: 73c06bb6-1195-4d3a-9a58-d01702b61f73 kind: Event lastTimestamp: "2026-09-16T19:06:01Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1986" already present on machine metadata: creationTimestamp: "2026-09-16T19:06:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:06:01Z" name: success-200-isvc-f64b9-predictor-ffc69556f-g6c6n.18d5e29642de108b namespace: kserve-ci-e2e-test resourceVersion: "20472" uid: daddec45-83aa-4ecd-9396-382f7c30bc15 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f64b9-predictor-ffc69556f-g6c6n namespace: kserve-ci-e2e-test resourceVersion: "20410" uid: 73c06bb6-1195-4d3a-9a58-d01702b61f73 kind: Event lastTimestamp: "2026-09-16T19:06:01Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T19:06:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:06:01Z" name: success-200-isvc-f64b9-predictor-ffc69556f-g6c6n.18d5e29645d768b7 namespace: kserve-ci-e2e-test resourceVersion: "20473" uid: 992eb1b6-634a-46dc-810c-2de9872f61e2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f64b9-predictor-ffc69556f-g6c6n namespace: kserve-ci-e2e-test resourceVersion: "20410" uid: 73c06bb6-1195-4d3a-9a58-d01702b61f73 kind: Event lastTimestamp: "2026-09-16T19:06:02Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T19:06:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:06:02Z" name: success-200-isvc-f64b9-predictor-ffc69556f-g6c6n.18d5e296464ae9f3 namespace: kserve-ci-e2e-test resourceVersion: "20474" uid: 03d33631-05bd-49ca-900e-33fccc4ed3c0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f64b9-predictor-ffc69556f-g6c6n namespace: kserve-ci-e2e-test resourceVersion: "20410" uid: 73c06bb6-1195-4d3a-9a58-d01702b61f73 kind: Event lastTimestamp: "2026-09-16T19:06:02Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-16T19:06:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:06:02Z" name: success-200-isvc-f64b9-predictor-ffc69556f-g6c6n.18d5e296465520c7 namespace: kserve-ci-e2e-test resourceVersion: "20475" uid: f5e60338-02f2-4339-a7d5-198a87bbbe71 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f64b9-predictor-ffc69556f-g6c6n namespace: kserve-ci-e2e-test resourceVersion: "20410" uid: 73c06bb6-1195-4d3a-9a58-d01702b61f73 kind: Event lastTimestamp: "2026-09-16T19:06:02Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T19:06:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:06:02Z" name: success-200-isvc-f64b9-predictor-ffc69556f-g6c6n.18d5e296497c18b6 namespace: kserve-ci-e2e-test resourceVersion: "20476" uid: bd835041-2ce0-4e1c-87de-189098aea0a4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f64b9-predictor-ffc69556f-g6c6n namespace: kserve-ci-e2e-test resourceVersion: "20410" uid: 73c06bb6-1195-4d3a-9a58-d01702b61f73 kind: Event lastTimestamp: "2026-09-16T19:06:02Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:06:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:06:02Z" name: success-200-isvc-f64b9-predictor-ffc69556f-g6c6n.18d5e29649f10c6f namespace: kserve-ci-e2e-test resourceVersion: "20477" uid: dd4af15a-71c7-4606-8064-d8b71c489883 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T19:06:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f64b9-predictor-ffc69556f-g6c6n namespace: kserve-ci-e2e-test resourceVersion: "20410" uid: 73c06bb6-1195-4d3a-9a58-d01702b61f73 kind: Event lastTimestamp: "2026-09-16T19:06:39Z" message: 'Readiness probe failed: dial tcp 10.134.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:06:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:06:39Z" name: success-200-isvc-f64b9-predictor-ffc69556f-g6c6n.18d5e296b738ac9a namespace: kserve-ci-e2e-test resourceVersion: "21005" uid: c4b6542e-52ee-4a8d-99ec-313dfb3b8aea reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f64b9-predictor-ffc69556f-g6c6n namespace: kserve-ci-e2e-test resourceVersion: "20410" uid: 73c06bb6-1195-4d3a-9a58-d01702b61f73 kind: Event lastTimestamp: "2026-09-16T19:07:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T19:07:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:10Z" name: success-200-isvc-f64b9-predictor-ffc69556f-g6c6n.18d5e2a652b63614 namespace: kserve-ci-e2e-test resourceVersion: "21351" uid: 47c09c3b-a30f-459a-9d70-320a1f9c430d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f64b9-predictor-ffc69556f-g6c6n namespace: kserve-ci-e2e-test resourceVersion: "20410" uid: 73c06bb6-1195-4d3a-9a58-d01702b61f73 kind: Event lastTimestamp: "2026-09-16T19:07:10Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:07:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:10Z" name: success-200-isvc-f64b9-predictor-ffc69556f-g6c6n.18d5e2a652b706db namespace: kserve-ci-e2e-test resourceVersion: "21358" uid: dfaaaa89-d392-4e40-8ed8-cd43d64bbd2d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-26.ec2.internal source: component: kubelet host: ip-10-0-130-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-f64b9-predictor-ffc69556f namespace: kserve-ci-e2e-test resourceVersion: "20395" uid: c1be4924-20bd-4149-86d9-68e0991f8164 kind: Event lastTimestamp: "2026-09-16T19:06:00Z" message: 'Created pod: success-200-isvc-f64b9-predictor-ffc69556f-g6c6n' metadata: creationTimestamp: "2026-09-16T19:06:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:06:00Z" name: success-200-isvc-f64b9-predictor-ffc69556f.18d5e295fab55bc1 namespace: kserve-ci-e2e-test resourceVersion: "20409" uid: 728a630e-060c-46e9-afe9-66f94685c8d7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-f64b9-predictor namespace: kserve-ci-e2e-test resourceVersion: "20391" uid: 343ce51b-71bb-4d77-b2e7-b71d000e0038 kind: Event lastTimestamp: "2026-09-16T19:06:00Z" message: Scaled up replica set success-200-isvc-f64b9-predictor-ffc69556f from 0 to 1 metadata: creationTimestamp: "2026-09-16T19:06:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:06:00Z" name: success-200-isvc-f64b9-predictor.18d5e295f835f42b namespace: kserve-ci-e2e-test resourceVersion: "20397" uid: 508fa558-cd36-4766-9af4-9a075b90bf7b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:06:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f64b9-predictor namespace: kserve-ci-e2e-test resourceVersion: "20394" uid: d9bb669b-31ee-43c3-b167-c35eb93d7d84 kind: Event lastTimestamp: "2026-09-16T19:06:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:06:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:06:30Z" name: success-200-isvc-f64b9-predictor.18d5e29976f4b81c namespace: kserve-ci-e2e-test resourceVersion: "20781" uid: f4639116-5615-4066-b68a-b8bbbe62175a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:06:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f64b9-predictor namespace: kserve-ci-e2e-test resourceVersion: "20394" uid: d9bb669b-31ee-43c3-b167-c35eb93d7d84 kind: Event lastTimestamp: "2026-09-16T19:06:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:06:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:06:30Z" name: success-200-isvc-f64b9-predictor.18d5e299775d38ae namespace: kserve-ci-e2e-test resourceVersion: "20782" uid: d0f1172e-de40-4efe-87ca-8bda50406164 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:06:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f64b9-predictor namespace: kserve-ci-e2e-test resourceVersion: "20616" uid: d9bb669b-31ee-43c3-b167-c35eb93d7d84 kind: Event lastTimestamp: "2026-09-16T19:07:00Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:06:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:07:00Z" name: success-200-isvc-f64b9-predictor.18d5e2a0765f614a namespace: kserve-ci-e2e-test resourceVersion: "21215" uid: 285d443c-afcd-44b1-9d41-98c56585421d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:06:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f64b9-predictor namespace: kserve-ci-e2e-test resourceVersion: "20616" uid: d9bb669b-31ee-43c3-b167-c35eb93d7d84 kind: Event lastTimestamp: "2026-09-16T19:07:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:06:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:07:00Z" name: success-200-isvc-f64b9-predictor.18d5e2a0775e15bf namespace: kserve-ci-e2e-test resourceVersion: "21218" uid: df6af92b-2986-4128-b4cd-d8397bb95fa2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T19:06:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f64b9 namespace: kserve-ci-e2e-test resourceVersion: "20357" uid: 727df9f6-6479-4cb8-a289-3329139e98c9 kind: Event lastTimestamp: "2026-09-16T19:06:50Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T19:06:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:06:50Z" name: success-200-isvc-f64b9.18d5e295f83e0bd8 namespace: kserve-ci-e2e-test resourceVersion: "21110" uid: a14cd447-2553-4ac7-b857-169cea67ca42 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f64b9 namespace: kserve-ci-e2e-test resourceVersion: "20357" uid: 727df9f6-6479-4cb8-a289-3329139e98c9 kind: Event lastTimestamp: "2026-09-16T19:06:00Z" message: 'Failed to update status for InferenceService "success-200-isvc-f64b9": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-f64b9": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-16T19:06:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:06:00Z" name: success-200-isvc-f64b9.18d5e295f8b11c2a namespace: kserve-ci-e2e-test resourceVersion: "20401" uid: 55c53f34-73da-4ec0-9a64-a66673e27760 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f64b9 namespace: kserve-ci-e2e-test resourceVersion: "20357" uid: 727df9f6-6479-4cb8-a289-3329139e98c9 kind: Event lastTimestamp: "2026-09-16T19:06:00Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-f64b9": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-16T19:06:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:06:00Z" name: success-200-isvc-f64b9.18d5e295f8b15b8e namespace: kserve-ci-e2e-test resourceVersion: "20404" uid: 90a0ec48-9648-404c-9895-57620184e520 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:06:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f64b9 namespace: kserve-ci-e2e-test resourceVersion: "21108" uid: 727df9f6-6479-4cb8-a289-3329139e98c9 kind: Event lastTimestamp: "2026-09-16T19:06:50Z" message: InferenceService [success-200-isvc-f64b9] is Ready metadata: creationTimestamp: "2026-09-16T19:06:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:06:50Z" name: success-200-isvc-f64b9.18d5e2a173853751 namespace: kserve-ci-e2e-test resourceVersion: "21109" uid: 6027c152-7edd-401b-a43b-60fc355681c4 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T18:55:55.622806Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-aa795-cf8569c7b-m8kxr namespace: kserve-ci-e2e-test resourceVersion: "15402" uid: 767daf61-7d49-4541-9057-b60f19d2d7e4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-aa795-cf8569c7b-m8kxr to ip-10-0-129-152.ec2.internal metadata: creationTimestamp: "2026-09-16T18:55:55Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T18:55:55Z" name: switch-graph-aa795-cf8569c7b-m8kxr.18d5e2091747eb02 namespace: kserve-ci-e2e-test resourceVersion: "15407" uid: d92f1e7c-2a8b-418a-b2b8-3d72c1706592 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-594cc48779-9d5dg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:55:56Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-aa795-cf8569c7b-m8kxr namespace: kserve-ci-e2e-test resourceVersion: "15410" uid: 767daf61-7d49-4541-9057-b60f19d2d7e4 kind: Event lastTimestamp: "2026-09-16T18:55:56Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T18:55:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T18:55:56Z" name: switch-graph-aa795-cf8569c7b-m8kxr.18d5e209391bb753 namespace: kserve-ci-e2e-test resourceVersion: "15432" uid: 0ce46ecb-3a62-44e2-a580-92c1f0a91a02 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:55:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-aa795} kind: Pod name: switch-graph-aa795-cf8569c7b-m8kxr namespace: kserve-ci-e2e-test resourceVersion: "15405" uid: 767daf61-7d49-4541-9057-b60f19d2d7e4 kind: Event lastTimestamp: "2026-09-16T18:55:56Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:10ee05d4acb8a93a3eeee1c000e44a3fdd4ba5c5852a56e3b2d09fe8b08cf42b" metadata: creationTimestamp: "2026-09-16T18:55:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:55:56Z" name: switch-graph-aa795-cf8569c7b-m8kxr.18d5e2093a9334a3 namespace: kserve-ci-e2e-test resourceVersion: "15435" uid: 5505bdf3-5180-4d94-88ab-a9f01d1fee13 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-152.ec2.internal source: component: kubelet host: ip-10-0-129-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:55:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-aa795} kind: Pod name: switch-graph-aa795-cf8569c7b-m8kxr namespace: kserve-ci-e2e-test resourceVersion: "15405" uid: 767daf61-7d49-4541-9057-b60f19d2d7e4 kind: Event lastTimestamp: "2026-09-16T18:55:58Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:10ee05d4acb8a93a3eeee1c000e44a3fdd4ba5c5852a56e3b2d09fe8b08cf42b" in 1.977s (1.977s including waiting). Image size: 211458663 bytes.' metadata: creationTimestamp: "2026-09-16T18:55:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:55:58Z" name: switch-graph-aa795-cf8569c7b-m8kxr.18d5e209b0743577 namespace: kserve-ci-e2e-test resourceVersion: "15454" uid: d16f6049-6463-409a-9975-d6615aaf2bc6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-152.ec2.internal source: component: kubelet host: ip-10-0-129-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:55:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-aa795} kind: Pod name: switch-graph-aa795-cf8569c7b-m8kxr namespace: kserve-ci-e2e-test resourceVersion: "15405" uid: 767daf61-7d49-4541-9057-b60f19d2d7e4 kind: Event lastTimestamp: "2026-09-16T18:55:58Z" message: 'Created container: switch-graph-aa795' metadata: creationTimestamp: "2026-09-16T18:55:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:55:58Z" name: switch-graph-aa795-cf8569c7b-m8kxr.18d5e209b374fea6 namespace: kserve-ci-e2e-test resourceVersion: "15457" uid: 4b33a56a-e565-41f4-84f6-158ca12d4fba reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-152.ec2.internal source: component: kubelet host: ip-10-0-129-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:55:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-aa795} kind: Pod name: switch-graph-aa795-cf8569c7b-m8kxr namespace: kserve-ci-e2e-test resourceVersion: "15405" uid: 767daf61-7d49-4541-9057-b60f19d2d7e4 kind: Event lastTimestamp: "2026-09-16T18:55:58Z" message: Started container switch-graph-aa795 metadata: creationTimestamp: "2026-09-16T18:55:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:55:58Z" name: switch-graph-aa795-cf8569c7b-m8kxr.18d5e209b3e94fb8 namespace: kserve-ci-e2e-test resourceVersion: "15458" uid: f0187b0c-14d5-4e7e-aa77-c9795f6210fa reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-152.ec2.internal source: component: kubelet host: ip-10-0-129-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:56:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-aa795} kind: Pod name: switch-graph-aa795-cf8569c7b-m8kxr namespace: kserve-ci-e2e-test resourceVersion: "15405" uid: 767daf61-7d49-4541-9057-b60f19d2d7e4 kind: Event lastTimestamp: "2026-09-16T18:56:05Z" message: Stopping container switch-graph-aa795 metadata: creationTimestamp: "2026-09-16T18:56:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:05Z" name: switch-graph-aa795-cf8569c7b-m8kxr.18d5e20b6e98f469 namespace: kserve-ci-e2e-test resourceVersion: "15532" uid: a02fc579-ad76-4511-9a9e-ecd6f8d7926e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-152.ec2.internal source: component: kubelet host: ip-10-0-129-152.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T18:56:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-aa795} kind: Pod name: switch-graph-aa795-cf8569c7b-m8kxr namespace: kserve-ci-e2e-test resourceVersion: "15405" uid: 767daf61-7d49-4541-9057-b60f19d2d7e4 kind: Event lastTimestamp: "2026-09-16T18:56:34Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-16T18:56:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:56:34Z" name: switch-graph-aa795-cf8569c7b-m8kxr.18d5e20c4ab84ea5 namespace: kserve-ci-e2e-test resourceVersion: "15913" uid: f12073ca-9179-493a-a066-6e3caf9bfe66 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-152.ec2.internal source: component: kubelet host: ip-10-0-129-152.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:55:55Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-aa795-cf8569c7b namespace: kserve-ci-e2e-test resourceVersion: "15398" uid: 3563684b-9c0f-4b8e-9a70-dd121537197d kind: Event lastTimestamp: "2026-09-16T18:55:55Z" message: 'Created pod: switch-graph-aa795-cf8569c7b-m8kxr' metadata: creationTimestamp: "2026-09-16T18:55:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:55:55Z" name: switch-graph-aa795-cf8569c7b.18d5e20916357891 namespace: kserve-ci-e2e-test resourceVersion: "15404" uid: 7398df16-023a-45e5-a697-c6eccff662b4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:55:55Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-aa795 namespace: kserve-ci-e2e-test resourceVersion: "15396" uid: 5f398a57-f357-45be-9440-8eed0637ffa5 kind: Event lastTimestamp: "2026-09-16T18:55:55Z" message: Scaled up replica set switch-graph-aa795-cf8569c7b from 0 to 1 metadata: creationTimestamp: "2026-09-16T18:55:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:55:55Z" name: switch-graph-aa795.18d5e20912e603f7 namespace: kserve-ci-e2e-test resourceVersion: "15400" uid: fb955d1e-c0c1-42af-8076-a5ae10a8d844 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:55:55Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-aa795 namespace: kserve-ci-e2e-test resourceVersion: "15412" uid: d577620a-fc7e-4e0b-9a92-7b5e25fa7679 kind: Event lastTimestamp: "2026-09-16T18:55:55Z" message: InferenceGraph [switch-graph-aa795] is Ready metadata: creationTimestamp: "2026-09-16T18:55:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T18:55:55Z" name: switch-graph-aa795.18d5e2091c1d6121 namespace: kserve-ci-e2e-test resourceVersion: "15414" uid: eb10e058-f94d-4a49-8989-97a91e953cd4 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:17:25.900740Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-ca22e-64559f96c5-kscpd namespace: kserve-ci-e2e-test resourceVersion: "26760" uid: 359602d5-6cff-45e4-9e3d-f500149e3eab kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-ca22e-64559f96c5-kscpd to ip-10-0-129-152.ec2.internal metadata: creationTimestamp: "2026-09-16T19:17:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:17:25Z" name: switch-graph-ca22e-64559f96c5-kscpd.18d5e33581d6bafa namespace: kserve-ci-e2e-test resourceVersion: "26766" uid: a5a3c985-d2d6-4039-a43b-15f195a8d854 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-594cc48779-9d5dg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:17:26Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-ca22e-64559f96c5-kscpd namespace: kserve-ci-e2e-test resourceVersion: "26763" uid: 359602d5-6cff-45e4-9e3d-f500149e3eab kind: Event lastTimestamp: "2026-09-16T19:17:26Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-ca22e-serving-cert" not found' metadata: creationTimestamp: "2026-09-16T19:17:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:17:26Z" name: switch-graph-ca22e-64559f96c5-kscpd.18d5e3358eb45112 namespace: kserve-ci-e2e-test resourceVersion: "26781" uid: eb8fb939-9b10-4e23-b0e3-53d9828f2cd0 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-152.ec2.internal source: component: kubelet host: ip-10-0-129-152.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:17:27Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-ca22e-64559f96c5-kscpd namespace: kserve-ci-e2e-test resourceVersion: "26769" uid: 359602d5-6cff-45e4-9e3d-f500149e3eab kind: Event lastTimestamp: "2026-09-16T19:17:27Z" message: Add eth0 [10.133.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:17:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:17:27Z" name: switch-graph-ca22e-64559f96c5-kscpd.18d5e335c7961c78 namespace: kserve-ci-e2e-test resourceVersion: "26798" uid: b54a5a10-ab76-471b-8720-8153ab4fdf39 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:17:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-ca22e} kind: Pod name: switch-graph-ca22e-64559f96c5-kscpd namespace: kserve-ci-e2e-test resourceVersion: "26763" uid: 359602d5-6cff-45e4-9e3d-f500149e3eab kind: Event lastTimestamp: "2026-09-16T19:17:27Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:10ee05d4acb8a93a3eeee1c000e44a3fdd4ba5c5852a56e3b2d09fe8b08cf42b" already present on machine metadata: creationTimestamp: "2026-09-16T19:17:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:17:27Z" name: switch-graph-ca22e-64559f96c5-kscpd.18d5e335c8f795f8 namespace: kserve-ci-e2e-test resourceVersion: "26800" uid: 934cc696-6b09-4c3e-ae8c-8803f34417d9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-152.ec2.internal source: component: kubelet host: ip-10-0-129-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:17:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-ca22e} kind: Pod name: switch-graph-ca22e-64559f96c5-kscpd namespace: kserve-ci-e2e-test resourceVersion: "26763" uid: 359602d5-6cff-45e4-9e3d-f500149e3eab kind: Event lastTimestamp: "2026-09-16T19:17:27Z" message: 'Created container: switch-graph-ca22e' metadata: creationTimestamp: "2026-09-16T19:17:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:17:27Z" name: switch-graph-ca22e-64559f96c5-kscpd.18d5e335ccdfabaa namespace: kserve-ci-e2e-test resourceVersion: "26801" uid: ef94b66d-eee0-42b3-aab8-63da7506d2d1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-152.ec2.internal source: component: kubelet host: ip-10-0-129-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:17:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-ca22e} kind: Pod name: switch-graph-ca22e-64559f96c5-kscpd namespace: kserve-ci-e2e-test resourceVersion: "26763" uid: 359602d5-6cff-45e4-9e3d-f500149e3eab kind: Event lastTimestamp: "2026-09-16T19:17:27Z" message: Started container switch-graph-ca22e metadata: creationTimestamp: "2026-09-16T19:17:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:17:27Z" name: switch-graph-ca22e-64559f96c5-kscpd.18d5e335cd5148e8 namespace: kserve-ci-e2e-test resourceVersion: "26802" uid: e670ec8b-f718-4470-85ab-a2f390b46522 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-152.ec2.internal source: component: kubelet host: ip-10-0-129-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:33:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-ca22e} kind: Pod name: switch-graph-ca22e-64559f96c5-kscpd namespace: kserve-ci-e2e-test resourceVersion: "26763" uid: 359602d5-6cff-45e4-9e3d-f500149e3eab kind: Event lastTimestamp: "2026-09-16T19:33:45Z" message: Stopping container switch-graph-ca22e metadata: creationTimestamp: "2026-09-16T19:33:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:33:45Z" name: switch-graph-ca22e-64559f96c5-kscpd.18d5e41984e2c0c6 namespace: kserve-ci-e2e-test resourceVersion: "33662" uid: 70ffc74a-9b40-4d74-9d70-9d3fa14f734d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-152.ec2.internal source: component: kubelet host: ip-10-0-129-152.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T19:33:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-ca22e} kind: Pod name: switch-graph-ca22e-64559f96c5-kscpd namespace: kserve-ci-e2e-test resourceVersion: "26763" uid: 359602d5-6cff-45e4-9e3d-f500149e3eab kind: Event lastTimestamp: "2026-09-16T19:34:13Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-16T19:33:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:34:13Z" name: switch-graph-ca22e-64559f96c5-kscpd.18d5e41a40f5ab0a namespace: kserve-ci-e2e-test resourceVersion: "33944" uid: 4a828da8-c78d-4535-bea3-a0b74c22f1a1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-152.ec2.internal source: component: kubelet host: ip-10-0-129-152.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:17:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-ca22e-64559f96c5 namespace: kserve-ci-e2e-test resourceVersion: "26757" uid: ba1a2a42-112e-40c9-9ae8-b010b1f361d0 kind: Event lastTimestamp: "2026-09-16T19:17:25Z" message: 'Created pod: switch-graph-ca22e-64559f96c5-kscpd' metadata: creationTimestamp: "2026-09-16T19:17:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:17:25Z" name: switch-graph-ca22e-64559f96c5.18d5e33581321876 namespace: kserve-ci-e2e-test resourceVersion: "26761" uid: 4ad5e1aa-c6f7-4fce-bedf-70c1dc2b9acc reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:17:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-ca22e namespace: kserve-ci-e2e-test resourceVersion: "26755" uid: 35218225-eb81-4c48-8a68-459141ba0bc5 kind: Event lastTimestamp: "2026-09-16T19:17:25Z" message: Scaled up replica set switch-graph-ca22e-64559f96c5 from 0 to 1 metadata: creationTimestamp: "2026-09-16T19:17:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:17:25Z" name: switch-graph-ca22e.18d5e33580648b4f namespace: kserve-ci-e2e-test resourceVersion: "26758" uid: 6d41ac8e-dd51-41cb-90ca-7e48aef866d8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:17:25Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-ca22e namespace: kserve-ci-e2e-test resourceVersion: "26773" uid: 185c5fa8-cfb5-4f81-9aa4-67a6e6997087 kind: Event lastTimestamp: "2026-09-16T19:17:25Z" message: InferenceGraph [switch-graph-ca22e] is Ready metadata: creationTimestamp: "2026-09-16T19:17:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:17:25Z" name: switch-graph-ca22e.18d5e3358751598a namespace: kserve-ci-e2e-test resourceVersion: "26775" uid: a5616e67-de07-4de0-9d16-9b460ba58dba reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T19:17:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-ca22e namespace: kserve-ci-e2e-test resourceVersion: "26756" uid: c2504d7e-e603-41f3-83e6-87dcf402b21c kind: Event lastTimestamp: "2026-09-16T19:18:10Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:17:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:18:10Z" name: switch-graph-ca22e.18d5e338fee579ec namespace: kserve-ci-e2e-test resourceVersion: "27203" uid: c134720f-d06b-4ab7-a04a-666371f17dea reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T19:17:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-ca22e namespace: kserve-ci-e2e-test resourceVersion: "26756" uid: c2504d7e-e603-41f3-83e6-87dcf402b21c kind: Event lastTimestamp: "2026-09-16T19:18:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:17:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:18:10Z" name: switch-graph-ca22e.18d5e338ff5a4e16 namespace: kserve-ci-e2e-test resourceVersion: "27204" uid: 63963a01-3936-4594-bdaf-7e7f2e87c2a0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-16T18:57:05.936581Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-eb09c-59454c5945-29wwk namespace: kserve-ci-e2e-test resourceVersion: "16405" uid: feeb6a12-6d98-4ba8-8358-7955557a62b1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-eb09c-59454c5945-29wwk to ip-10-0-129-152.ec2.internal metadata: creationTimestamp: "2026-09-16T18:57:05Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T18:57:05Z" name: switch-graph-eb09c-59454c5945-29wwk.18d5e219764ef97e namespace: kserve-ci-e2e-test resourceVersion: "16410" uid: be12446b-2095-49f2-83a7-1e22c4faa4b6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-594cc48779-9d5dg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:06Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-eb09c-59454c5945-29wwk namespace: kserve-ci-e2e-test resourceVersion: "16408" uid: feeb6a12-6d98-4ba8-8358-7955557a62b1 kind: Event lastTimestamp: "2026-09-16T18:57:06Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-eb09c-serving-cert" not found' metadata: creationTimestamp: "2026-09-16T18:57:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:57:06Z" name: switch-graph-eb09c-59454c5945-29wwk.18d5e21980ac07f8 namespace: kserve-ci-e2e-test resourceVersion: "16424" uid: 05a83ac9-11f9-4c1d-99a8-c29a69c65727 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-152.ec2.internal source: component: kubelet host: ip-10-0-129-152.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:07Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-eb09c-59454c5945-29wwk namespace: kserve-ci-e2e-test resourceVersion: "16413" uid: feeb6a12-6d98-4ba8-8358-7955557a62b1 kind: Event lastTimestamp: "2026-09-16T18:57:07Z" message: Add eth0 [10.133.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T18:57:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T18:57:07Z" name: switch-graph-eb09c-59454c5945-29wwk.18d5e219bbe29578 namespace: kserve-ci-e2e-test resourceVersion: "16436" uid: 70c57ce4-90c6-46c0-a7fd-e3886b8d570f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-eb09c} kind: Pod name: switch-graph-eb09c-59454c5945-29wwk namespace: kserve-ci-e2e-test resourceVersion: "16408" uid: feeb6a12-6d98-4ba8-8358-7955557a62b1 kind: Event lastTimestamp: "2026-09-16T18:57:07Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:10ee05d4acb8a93a3eeee1c000e44a3fdd4ba5c5852a56e3b2d09fe8b08cf42b" already present on machine metadata: creationTimestamp: "2026-09-16T18:57:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:57:07Z" name: switch-graph-eb09c-59454c5945-29wwk.18d5e219bd2c767e namespace: kserve-ci-e2e-test resourceVersion: "16438" uid: 9ec2c375-5f98-4b28-b918-5aaaaabf878a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-152.ec2.internal source: component: kubelet host: ip-10-0-129-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-eb09c} kind: Pod name: switch-graph-eb09c-59454c5945-29wwk namespace: kserve-ci-e2e-test resourceVersion: "16408" uid: feeb6a12-6d98-4ba8-8358-7955557a62b1 kind: Event lastTimestamp: "2026-09-16T18:57:07Z" message: 'Created container: switch-graph-eb09c' metadata: creationTimestamp: "2026-09-16T18:57:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:57:07Z" name: switch-graph-eb09c-59454c5945-29wwk.18d5e219c0cece0f namespace: kserve-ci-e2e-test resourceVersion: "16439" uid: 5e86402b-7e34-496b-8dea-6b51a161e56e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-152.ec2.internal source: component: kubelet host: ip-10-0-129-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-eb09c} kind: Pod name: switch-graph-eb09c-59454c5945-29wwk namespace: kserve-ci-e2e-test resourceVersion: "16408" uid: feeb6a12-6d98-4ba8-8358-7955557a62b1 kind: Event lastTimestamp: "2026-09-16T18:57:07Z" message: Started container switch-graph-eb09c metadata: creationTimestamp: "2026-09-16T18:57:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:57:07Z" name: switch-graph-eb09c-59454c5945-29wwk.18d5e219c1424741 namespace: kserve-ci-e2e-test resourceVersion: "16440" uid: 1ff435b7-ed0f-41f3-b389-58941e251992 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-152.ec2.internal source: component: kubelet host: ip-10-0-129-152.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:05:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-eb09c} kind: Pod name: switch-graph-eb09c-59454c5945-29wwk namespace: kserve-ci-e2e-test resourceVersion: "16408" uid: feeb6a12-6d98-4ba8-8358-7955557a62b1 kind: Event lastTimestamp: "2026-09-16T19:05:20Z" message: Stopping container switch-graph-eb09c metadata: creationTimestamp: "2026-09-16T19:05:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:05:20Z" name: switch-graph-eb09c-59454c5945-29wwk.18d5e28c9bfcef7e namespace: kserve-ci-e2e-test resourceVersion: "19903" uid: db9bbaf7-d0ff-489b-954b-a3eb1e478e9d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-152.ec2.internal source: component: kubelet host: ip-10-0-129-152.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T19:05:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-eb09c} kind: Pod name: switch-graph-eb09c-59454c5945-29wwk namespace: kserve-ci-e2e-test resourceVersion: "16408" uid: feeb6a12-6d98-4ba8-8358-7955557a62b1 kind: Event lastTimestamp: "2026-09-16T19:05:48Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-16T19:05:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:05:48Z" name: switch-graph-eb09c-59454c5945-29wwk.18d5e28d5a7ca2c1 namespace: kserve-ci-e2e-test resourceVersion: "20245" uid: 39931139-aabc-47ac-981c-dd78e01b6cba reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-152.ec2.internal source: component: kubelet host: ip-10-0-129-152.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:05Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-eb09c-59454c5945 namespace: kserve-ci-e2e-test resourceVersion: "16402" uid: 64555724-dcd6-48b9-845a-fe53cd397168 kind: Event lastTimestamp: "2026-09-16T18:57:05Z" message: 'Created pod: switch-graph-eb09c-59454c5945-29wwk' metadata: creationTimestamp: "2026-09-16T18:57:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:57:05Z" name: switch-graph-eb09c-59454c5945.18d5e21975d597ff namespace: kserve-ci-e2e-test resourceVersion: "16407" uid: edc62983-7fba-4419-a512-894da081ff88 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:05Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-eb09c namespace: kserve-ci-e2e-test resourceVersion: "16400" uid: edaa47b8-a836-405f-9b17-03ffb2b13a56 kind: Event lastTimestamp: "2026-09-16T18:57:05Z" message: Scaled up replica set switch-graph-eb09c-59454c5945 from 0 to 1 metadata: creationTimestamp: "2026-09-16T18:57:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:57:05Z" name: switch-graph-eb09c.18d5e219745fa607 namespace: kserve-ci-e2e-test resourceVersion: "16404" uid: 96f2fcdf-9a03-4a3f-9f40-d6e2fb31d489 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:06Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-eb09c namespace: kserve-ci-e2e-test resourceVersion: "16418" uid: 116257e9-97fb-418e-a655-c2733bc2671f kind: Event lastTimestamp: "2026-09-16T18:57:06Z" message: InferenceGraph [switch-graph-eb09c] is Ready metadata: creationTimestamp: "2026-09-16T18:57:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T18:57:06Z" name: switch-graph-eb09c.18d5e2197b58fcf0 namespace: kserve-ci-e2e-test resourceVersion: "16420" uid: 3d3876f7-2397-4d47-91fc-92c5665319d4 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-eb09c namespace: kserve-ci-e2e-test resourceVersion: "16401" uid: c9501d6f-2dc1-4194-a5c2-b7c5deb85a43 kind: Event lastTimestamp: "2026-09-16T18:57:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T18:57:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:57:20Z" name: switch-graph-eb09c.18d5e21cf2dc766a namespace: kserve-ci-e2e-test resourceVersion: "16554" uid: 3bcdac71-e8ea-44e7-ae51-28d54bed1eeb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-eb09c namespace: kserve-ci-e2e-test resourceVersion: "16401" uid: c9501d6f-2dc1-4194-a5c2-b7c5deb85a43 kind: Event lastTimestamp: "2026-09-16T18:57:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T18:57:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:57:20Z" name: switch-graph-eb09c.18d5e21cf3429ec7 namespace: kserve-ci-e2e-test resourceVersion: "16556" uid: 652eace9-2bfb-4be4-8b3a-4e92d5617518 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-eb09c namespace: kserve-ci-e2e-test resourceVersion: "16555" uid: c9501d6f-2dc1-4194-a5c2-b7c5deb85a43 kind: Event lastTimestamp: "2026-09-16T18:57:35Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T18:57:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:57:35Z" name: switch-graph-eb09c.18d5e22071fdcaaf namespace: kserve-ci-e2e-test resourceVersion: "16685" uid: b82437a5-81f5-4fac-bac6-64a92d4b2c46 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-eb09c namespace: kserve-ci-e2e-test resourceVersion: "16555" uid: c9501d6f-2dc1-4194-a5c2-b7c5deb85a43 kind: Event lastTimestamp: "2026-09-16T18:57:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T18:57:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:57:35Z" name: switch-graph-eb09c.18d5e220724d08c5 namespace: kserve-ci-e2e-test resourceVersion: "16687" uid: ed57d6a7-b961-4056-ba7e-13f089d433ae reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning kind: EventList metadata: resourceVersion: "33963"